diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/config/GeneralConfiguration.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/config/GeneralConfiguration.java index 245ca0ece..31cbc7bf9 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/config/GeneralConfiguration.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/config/GeneralConfiguration.java @@ -5,11 +5,13 @@ import fr.dossierfacile.common.utils.LocalDateTimeTypeAdapter; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.data.domain.AuditorAware; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.web.client.RestTemplate; import java.time.LocalDateTime; +import java.util.Optional; import java.util.concurrent.Executor; @Configuration @@ -43,4 +45,9 @@ public Gson gson() { builder.registerTypeAdapter(LocalDateTime.class, new LocalDateTimeTypeAdapter()); return builder.create(); } + + @Bean + public AuditorAware auditorAware() { + return (() -> Optional.of("api-tenant")); + } } diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java index 4865e4ff3..1b469458e 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java @@ -11,15 +11,14 @@ import fr.dossierfacile.common.entity.Document; import fr.dossierfacile.common.entity.File; import fr.dossierfacile.common.entity.Person; -import fr.dossierfacile.common.entity.StorageFile; import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.enums.DocumentCategory; import fr.dossierfacile.common.enums.DocumentStatus; import fr.dossierfacile.common.enums.TenantFileStatus; import fr.dossierfacile.common.model.log.EditionType; import fr.dossierfacile.common.repository.DocumentAnalysisReportRepository; -import fr.dossierfacile.common.repository.StorageFileRepository; import fr.dossierfacile.common.service.interfaces.DocumentHelperService; +import fr.dossierfacile.common.service.interfaces.FileStorageService; import fr.dossierfacile.common.service.interfaces.LogService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -42,7 +41,7 @@ public class DocumentServiceImpl implements DocumentService { private final DocumentRepository documentRepository; private final DocumentAnalysisReportRepository documentAnalysisReportRepository; - private final StorageFileRepository storageFileRepository; + private final FileStorageService fileStorageService; private final TenantStatusService tenantStatusService; private final ApartmentSharingService apartmentSharingService; private final DocumentHelperService documentHelperService; @@ -112,7 +111,7 @@ public void resetValidatedDocumentsStatusOfSpecifiedCategoriesToToProcess(List { @@ -148,8 +147,7 @@ public void markDocumentAsEdited(Document document) { document.setDocumentAnalysisReport(null); } if ( document.getWatermarkFile() != null ){ - StorageFile watermarkFile = document.getWatermarkFile(); - storageFileRepository.delete(watermarkFile); + fileStorageService.delete(document.getWatermarkFile()); document.setWatermarkFile(null); } documentRepository.save(document); diff --git a/dossierfacile-api-watermark/src/main/java/fr/dossierfacile/api/pdf/service/DocumentServiceImpl.java b/dossierfacile-api-watermark/src/main/java/fr/dossierfacile/api/pdf/service/DocumentServiceImpl.java index 8bddfacde..3f5300905 100644 --- a/dossierfacile-api-watermark/src/main/java/fr/dossierfacile/api/pdf/service/DocumentServiceImpl.java +++ b/dossierfacile-api-watermark/src/main/java/fr/dossierfacile/api/pdf/service/DocumentServiceImpl.java @@ -10,7 +10,6 @@ import fr.dossierfacile.common.entity.StorageFile; import fr.dossierfacile.common.entity.WatermarkDocument; import fr.dossierfacile.common.enums.FileStatus; -import fr.dossierfacile.common.repository.StorageFileRepository; import fr.dossierfacile.common.repository.WatermarkDocumentRepository; import fr.dossierfacile.common.service.interfaces.EncryptionKeyService; import fr.dossierfacile.common.service.interfaces.FileStorageService; @@ -39,7 +38,6 @@ public class DocumentServiceImpl implements DocumentService { private static final String DOCUMENT_NOT_EXIST = "The document does not exist"; private final Producer producer; private final FileStorageService fileStorageService; - private final StorageFileRepository storageFileRepository; private final EncryptionKeyService encryptionKeyService; private final WatermarkDocumentRepository watermarkDocumentRepository; @@ -130,9 +128,7 @@ private void cleanData(WatermarkDocument document) { document.setPdfFile(null); document.setPdfStatus(FileStatus.DELETED); watermarkDocumentRepository.save(document); - if (pdfFile != null) { - storageFileRepository.delete(pdfFile); - } + fileStorageService.delete(pdfFile); } @Override diff --git a/dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java b/dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java index 0da9ac3e7..ecaa29e56 100644 --- a/dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java +++ b/dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java @@ -7,7 +7,6 @@ import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.enums.DocumentStatus; import fr.dossierfacile.common.enums.DocumentSubCategory; -import fr.dossierfacile.common.repository.StorageFileRepository; import fr.dossierfacile.common.service.interfaces.FileStorageService; import fr.gouv.bo.amqp.Producer; import fr.gouv.bo.dto.MessageDTO; @@ -29,7 +28,6 @@ public class DocumentService { private final DocumentRepository documentRepository; private final FileStorageService fileStorageService; - private final StorageFileRepository storageFileRepository; private final Producer producer; private final DocumentDeniedOptionsRepository documentDeniedOptionsRepository; @@ -81,9 +79,7 @@ public void initializeFieldsToProcessPdfGeneration(Document document) { StorageFile watermarkFile = document.getWatermarkFile(); document.setWatermarkFile(null); documentRepository.save(document); - if (watermarkFile != null) { - storageFileRepository.delete(watermarkFile); - } + fileStorageService.delete(watermarkFile); } @Transactional diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ApartmentSharing.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ApartmentSharing.java index 87cb4dba3..d5982833e 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ApartmentSharing.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ApartmentSharing.java @@ -2,21 +2,9 @@ import fr.dossierfacile.common.enums.ApplicationType; import fr.dossierfacile.common.enums.FileStatus; +import fr.dossierfacile.common.enums.FileStorageStatus; import fr.dossierfacile.common.enums.TenantFileStatus; -import jakarta.persistence.CascadeType; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.EntityListeners; -import jakarta.persistence.EnumType; -import jakarta.persistence.Enumerated; -import jakarta.persistence.FetchType; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.OneToMany; -import jakarta.persistence.OneToOne; -import jakarta.persistence.Table; +import jakarta.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -64,7 +52,7 @@ public class ApartmentSharing implements Serializable { @Enumerated(EnumType.STRING) private ApplicationType applicationType; - @OneToOne(cascade = CascadeType.REMOVE, orphanRemoval = true) + @OneToOne @JoinColumn(name = "pdf_dossier_file_id") private StorageFile pdfDossierFile; @@ -78,6 +66,12 @@ public class ApartmentSharing implements Serializable { @OneToMany(mappedBy = "apartmentSharing", fetch = FetchType.LAZY, cascade = CascadeType.REMOVE) private List apartmentSharingLinks = new ArrayList<>(); + @PreRemove + void deleteCascade() { + if (pdfDossierFile != null) + pdfDossierFile.setStatus(FileStorageStatus.TO_DELETE); + } + public ApartmentSharing(Tenant tenant) { tenants.add(tenant); this.applicationType = ApplicationType.ALONE; diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/Document.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/Document.java index 00d27b778..80e3d3271 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/Document.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/Document.java @@ -3,20 +3,9 @@ import fr.dossierfacile.common.enums.DocumentCategory; import fr.dossierfacile.common.enums.DocumentStatus; import fr.dossierfacile.common.enums.DocumentSubCategory; +import fr.dossierfacile.common.enums.FileStorageStatus; import jakarta.annotation.Nullable; -import jakarta.persistence.CascadeType; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.EnumType; -import jakarta.persistence.Enumerated; -import jakarta.persistence.FetchType; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.ManyToOne; -import jakarta.persistence.OneToMany; -import jakarta.persistence.OneToOne; +import jakarta.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -89,7 +78,7 @@ public class Document implements Serializable { @Builder.Default private DocumentStatus documentStatus = DocumentStatus.TO_PROCESS; - @OneToOne(cascade = CascadeType.REMOVE, orphanRemoval = true) + @OneToOne @JoinColumn(name = "watermark_file_id") private StorageFile watermarkFile; @@ -105,6 +94,12 @@ public class Document implements Serializable { @OneToOne(mappedBy= "document", fetch = FetchType.LAZY, cascade = CascadeType.REMOVE) private DocumentAnalysisReport documentAnalysisReport; + @PreRemove + void deleteCascade() { + if (watermarkFile != null) + watermarkFile.setStatus(FileStorageStatus.TO_DELETE); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/File.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/File.java index b08e35ab2..59704fcc4 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/File.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/File.java @@ -1,16 +1,8 @@ package fr.dossierfacile.common.entity; +import fr.dossierfacile.common.enums.FileStorageStatus; import jakarta.annotation.Nullable; -import jakarta.persistence.CascadeType; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.FetchType; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.ManyToOne; -import jakarta.persistence.OneToOne; +import jakarta.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -18,6 +10,8 @@ import lombok.Setter; import lombok.ToString; import org.hibernate.Hibernate; +import org.hibernate.annotations.OnDelete; +import org.hibernate.annotations.SoftDelete; import java.io.Serial; import java.io.Serializable; @@ -40,13 +34,13 @@ public class File implements Serializable { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @OneToOne(cascade = CascadeType.REMOVE) + @OneToOne @JoinColumn(name = "storage_file_id") private StorageFile storageFile; private int numberOfPages; - @OneToOne(cascade = CascadeType.REMOVE) + @OneToOne @JoinColumn(name = "preview_file_id") private StorageFile preview; @@ -67,6 +61,15 @@ public class File implements Serializable { @OneToOne(mappedBy= "file", fetch = FetchType.LAZY) private ParsedFileAnalysis parsedFileAnalysis; + @PreRemove + void deleteCascade() { + if (storageFile != null) + storageFile.setStatus(FileStorageStatus.TO_DELETE); + + if (preview != null) + preview.setStatus(FileStorageStatus.TO_DELETE); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ObjectStorageProvider.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ObjectStorageProvider.java index bf6fe9428..d03933ccb 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ObjectStorageProvider.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ObjectStorageProvider.java @@ -2,5 +2,6 @@ public enum ObjectStorageProvider { OVH, - THREEDS_OUTSCALE + THREEDS_OUTSCALE, + LOCAL } \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/StorageFile.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/StorageFile.java index ed923a5c4..8c485df4c 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/StorageFile.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/StorageFile.java @@ -1,6 +1,7 @@ package fr.dossierfacile.common.entity; import fr.dossierfacile.common.entity.shared.AbstractAuditable; +import fr.dossierfacile.common.enums.FileStorageStatus; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.EnumType; @@ -45,6 +46,8 @@ public class StorageFile extends AbstractAuditable { protected String contentType; protected Long size; protected String md5; + @Enumerated(EnumType.STRING) + protected FileStorageStatus status; @Column @Enumerated(EnumType.STRING) protected ObjectStorageProvider provider; diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/StorageFileToDelete.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/StorageFileToDelete.java deleted file mode 100644 index e51e04335..000000000 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/StorageFileToDelete.java +++ /dev/null @@ -1,46 +0,0 @@ -package fr.dossierfacile.common.entity; - -import fr.dossierfacile.common.entity.shared.AbstractAuditable; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.Inheritance; -import jakarta.persistence.InheritanceType; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import lombok.experimental.SuperBuilder; - -import java.io.Serial; -import java.util.ArrayList; -import java.util.List; - -@Entity -@Inheritance(strategy = InheritanceType.JOINED) -@AllArgsConstructor -@NoArgsConstructor -@Getter -@Setter -@ToString -@SuperBuilder -public class StorageFileToDelete extends AbstractAuditable { - @Serial - private static final long serialVersionUID = 1L; - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - protected Long id; - - protected String path; - - @Column( - name = "providers", - columnDefinition = "character varying[]" - ) - protected List providers = new ArrayList<>(); - -} diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/WatermarkDocument.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/WatermarkDocument.java index a650bceed..800300d23 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/WatermarkDocument.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/WatermarkDocument.java @@ -1,19 +1,8 @@ package fr.dossierfacile.common.entity; import fr.dossierfacile.common.enums.FileStatus; -import jakarta.persistence.CascadeType; -import jakarta.persistence.Entity; -import jakarta.persistence.EntityListeners; -import jakarta.persistence.EnumType; -import jakarta.persistence.Enumerated; -import jakarta.persistence.FetchType; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.JoinColumn; -import jakarta.persistence.JoinTable; -import jakarta.persistence.OneToMany; -import jakarta.persistence.OneToOne; +import fr.dossierfacile.common.enums.FileStorageStatus; +import jakarta.persistence.*; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -58,9 +47,15 @@ public class WatermarkDocument { @Enumerated(EnumType.STRING) private FileStatus pdfStatus; - @OneToOne(cascade = CascadeType.REMOVE, orphanRemoval = true) + @OneToOne @JoinColumn(name = "pdf_file_id") private StorageFile pdfFile; private String text; + + @PreRemove + void deleteCascade() { + if (pdfFile != null) + pdfFile.setStatus(FileStorageStatus.TO_DELETE); + } } diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/FileStorageStatus.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/FileStorageStatus.java new file mode 100644 index 000000000..1f0eff3aa --- /dev/null +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/FileStorageStatus.java @@ -0,0 +1,8 @@ +package fr.dossierfacile.common.enums; + +public enum FileStorageStatus { + TO_DELETE, + COPY_FAILED, + DELETE_FAILED, + TEMPORARY; // should not be synchronized +} diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/interceptors/DeleteFileInterceptor.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/interceptors/DeleteFileInterceptor.java deleted file mode 100644 index ac7434122..000000000 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/interceptors/DeleteFileInterceptor.java +++ /dev/null @@ -1,50 +0,0 @@ -package fr.dossierfacile.common.interceptors; - -import fr.dossierfacile.common.entity.StorageFile; -import fr.dossierfacile.common.entity.StorageFileToDelete; -import fr.dossierfacile.common.repository.StorageFileToDeleteRepository; -import fr.dossierfacile.common.service.interfaces.FileStorageService; -import io.sentry.Sentry; -import lombok.extern.slf4j.Slf4j; -import org.hibernate.EmptyInterceptor; -import org.hibernate.type.Type; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.orm.jpa.HibernatePropertiesCustomizer; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Component; - -import java.io.Serializable; -import java.util.Map; - -@Component -@Slf4j -public class DeleteFileInterceptor extends EmptyInterceptor implements HibernatePropertiesCustomizer { - - @Autowired - @Lazy - private FileStorageService fileStorageService; - - @Autowired - @Lazy - private transient StorageFileToDeleteRepository storageFileToDeleteRepository; - - @Override - public void customize(Map hibernateProperties) { - hibernateProperties.put("hibernate.session_factory.interceptor", this); - } - - @Override - public void onDelete(Object entity, Serializable id, Object[] state, String[] propertyNames, Type[] types) { - try { - if (entity instanceof StorageFile storageFile) { - StorageFileToDelete storageFileToDelete = StorageFileToDelete.builder() - .providers(storageFile.getProviders()) - .path(storageFile.getPath()) - .build(); - storageFileToDeleteRepository.save(storageFileToDelete); - } - } catch (Throwable e) { - log.error("Unable to execute post delete operations! Sentry:" + Sentry.captureException(e), e); - } - } -} \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/StorageFileRepository.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/StorageFileRepository.java index 18f17876e..a3ec54d1b 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/StorageFileRepository.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/StorageFileRepository.java @@ -1,8 +1,11 @@ package fr.dossierfacile.common.repository; import fr.dossierfacile.common.entity.StorageFile; +import fr.dossierfacile.common.enums.FileStorageStatus; +import org.jetbrains.annotations.NotNull; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -16,14 +19,31 @@ public interface StorageFileRepository extends JpaRepository FROM storage_file sf WHERE array_length(sf.providers, 1) < 2 AND last_modified_date is not null - AND sf.last_modified_date < NOW() - INTERVAL '10' MINUTE + AND sf.last_modified_date < NOW() - INTERVAL '10' MINUTE AND sf.last_modified_date > NOW() - INTERVAL '10' DAY + AND sf.status is null ORDER BY sf.last_modified_date DESC """, nativeQuery = true) - List findAllWithOneProvider(Pageable pageable); + List findAllWithOneProviderAndReady(Pageable pageable); + + @Query(value = """ + SELECT * + FROM storage_file sf + WHERE array_length(sf.providers, 1) < 2 + AND last_modified_date is not null + AND sf.last_modified_date < NOW() - INTERVAL '10' MINUTE + AND sf.last_modified_date > NOW() - INTERVAL '10' DAY + AND sf.status = 'COPY_FAILED' + ORDER BY sf.last_modified_date DESC + """, nativeQuery = true) + List findAllWithOneProviderAndCopyFailed(Pageable pageable); + @Query(value = "SELECT path FROM storage_file WHERE path IN (:pathsToSearch)", nativeQuery = true) List findExistingPathsIn(@Param("pathsToSearch") List paths); + List findAllByStatus(FileStorageStatus fileStorageStatus); + + void delete(@NotNull StorageFile storageFile); } diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/StorageFileToDeleteRepository.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/StorageFileToDeleteRepository.java deleted file mode 100644 index f4c08a462..000000000 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/StorageFileToDeleteRepository.java +++ /dev/null @@ -1,7 +0,0 @@ -package fr.dossierfacile.common.repository; - -import fr.dossierfacile.common.entity.StorageFileToDelete; -import org.springframework.data.jpa.repository.JpaRepository; - -public interface StorageFileToDeleteRepository extends JpaRepository { -} \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/ApartmentSharingCommonServiceImpl.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/ApartmentSharingCommonServiceImpl.java index b7733955a..b67b6eb82 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/ApartmentSharingCommonServiceImpl.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/ApartmentSharingCommonServiceImpl.java @@ -33,14 +33,7 @@ public void resetDossierPdfGenerated(ApartmentSharing apartmentSharing) { apartmentSharing.setPdfDossierFile(null); apartmentSharing.setDossierPdfDocumentStatus(FileStatus.DELETED); apartmentSharingRepository.save(apartmentSharing); - - if (pdfFile != null) { - try { - fileStorageService.delete(pdfFile); - } catch (Exception e) { - log.error("Unable to delete pdfFile appartmentSharing:" + apartmentSharing.getId()); - } - } + fileStorageService.delete(pdfFile); } @Override @Transactional(propagation = Propagation.SUPPORTS) diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageServiceImpl.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageServiceImpl.java index fd915ca06..e6a96dce2 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageServiceImpl.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageServiceImpl.java @@ -3,57 +3,62 @@ import fr.dossierfacile.common.config.DynamicProviderConfig; import fr.dossierfacile.common.entity.ObjectStorageProvider; import fr.dossierfacile.common.entity.StorageFile; +import fr.dossierfacile.common.enums.FileStorageStatus; import fr.dossierfacile.common.exceptions.RetryableOperationException; import fr.dossierfacile.common.repository.StorageFileRepository; import fr.dossierfacile.common.service.interfaces.FileStorageProviderService; import fr.dossierfacile.common.service.interfaces.FileStorageService; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import java.io.IOException; import java.io.InputStream; import java.nio.file.ProviderNotFoundException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; +import java.util.*; @Service @Slf4j -@Profile("!mockOvh") +@AllArgsConstructor public class FileStorageServiceImpl implements FileStorageService { - private final FileStorageProviderService ovhFileStorageService; - private final FileStorageProviderService outscaleFileStorageService; private final StorageFileRepository storageFileRepository; private final DynamicProviderConfig dynamicProviderConfig; - public FileStorageServiceImpl(@Qualifier("ovhFileStorageProvider") FileStorageProviderService ovhFileStorageService, - @Qualifier("outscaleFileStorageProvider") FileStorageProviderService outscaleFileStorageService, - StorageFileRepository storageFileRepository, DynamicProviderConfig dynamicProviderConfig) { - this.ovhFileStorageService = ovhFileStorageService; - this.outscaleFileStorageService = outscaleFileStorageService; - this.storageFileRepository = storageFileRepository; - this.dynamicProviderConfig = dynamicProviderConfig; - } + private final List fileStorageProviders; private FileStorageProviderService getStorageService(ObjectStorageProvider storageProvider) { - return switch (storageProvider) { - case OVH -> ovhFileStorageService; - case THREEDS_OUTSCALE -> outscaleFileStorageService; - default -> throw new ProviderNotFoundException(); - }; + return fileStorageProviders.stream().filter(p -> p.getProvider() == storageProvider).findFirst().orElseThrow(() -> new ProviderNotFoundException()); } + /** + * Soft delete + */ @Override public void delete(StorageFile storageFile) { + if (storageFile != null) { + storageFile.setStatus(FileStorageStatus.TO_DELETE); + storageFileRepository.save(storageFile); + } + } + + @Override + public void hardDelete(StorageFile storageFile) { if (storageFile == null) { return; } - storageFileRepository.delete(storageFile); - getStorageService(storageFile.getProvider()).delete(storageFile.getPath()); + if (storageFile.getProviders() == null) { + storageFileRepository.delete(storageFile); + return; + } + try { + for (String provider : storageFile.getProviders()) { + getStorageService(ObjectStorageProvider.valueOf(provider)).delete(storageFile.getPath()); + } + storageFileRepository.delete(storageFile); + } catch (Exception e) { + storageFile.setStatus(FileStorageStatus.DELETE_FAILED); + storageFileRepository.save(storageFile); + } } @Override @@ -61,7 +66,7 @@ public InputStream download(StorageFile storageFile) throws IOException { List availableProviders = storageFile.getProviders(); for (ObjectStorageProvider provider : dynamicProviderConfig.getProviders()) { Optional selectedProvider = availableProviders.stream().filter(s -> Objects.equals(s, provider.name())).findAny(); - if(selectedProvider.isPresent()) { + if (selectedProvider.isPresent()) { try { return getStorageService(ObjectStorageProvider.valueOf(selectedProvider.get())) .download(storageFile.getPath(), storageFile.getEncryptionKey()); @@ -90,15 +95,15 @@ public StorageFile upload(InputStream inputStream, StorageFile storageFile) thro if (inputStream.markSupported()) { inputStream.mark(100000000); } - boolean shift=false; + boolean shift = false; for (ObjectStorageProvider provider : dynamicProviderConfig.getProviders()) { boolean tryNextProvider = false; try { storageFile = uploadToProvider(inputStream, storageFile, provider); } catch (RetryableOperationException e) { log.warn("Provider " + provider + " Failed - Retry with the next provider if exists.", e); - shift=true; - if(inputStream.markSupported()) { + shift = true; + if (inputStream.markSupported()) { inputStream.reset(); tryNextProvider = true; } diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageToDeleteServiceImpl.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageToDeleteServiceImpl.java deleted file mode 100644 index f0254e6f8..000000000 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageToDeleteServiceImpl.java +++ /dev/null @@ -1,57 +0,0 @@ -package fr.dossierfacile.common.service; - -import fr.dossierfacile.common.config.DynamicProviderConfig; -import fr.dossierfacile.common.entity.ObjectStorageProvider; -import fr.dossierfacile.common.entity.StorageFileToDelete; -import fr.dossierfacile.common.repository.StorageFileToDeleteRepository; -import fr.dossierfacile.common.service.interfaces.FileStorageProviderService; -import fr.dossierfacile.common.service.interfaces.FileStorageToDeleteService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Service; - -import java.nio.file.ProviderNotFoundException; - -@Service -@Slf4j -@Profile("!mockOvh") -public class FileStorageToDeleteServiceImpl implements FileStorageToDeleteService { - private final FileStorageProviderService ovhFileStorageService; - private final FileStorageProviderService outscaleFileStorageService; - private final StorageFileToDeleteRepository storageFileToDeleteRepository; - private final DynamicProviderConfig dynamicProviderConfig; - public FileStorageToDeleteServiceImpl(@Qualifier("ovhFileStorageProvider") FileStorageProviderService ovhFileStorageService, - @Qualifier("outscaleFileStorageProvider") FileStorageProviderService outscaleFileStorageService, - StorageFileToDeleteRepository storageFileToDeleteRepository, DynamicProviderConfig dynamicProviderConfig) { - this.ovhFileStorageService = ovhFileStorageService; - this.outscaleFileStorageService = outscaleFileStorageService; - this.storageFileToDeleteRepository = storageFileToDeleteRepository; - this.dynamicProviderConfig = dynamicProviderConfig; - } - - private FileStorageProviderService getStorageService(ObjectStorageProvider storageProvider) { - return switch (storageProvider) { - case OVH -> ovhFileStorageService; - case THREEDS_OUTSCALE -> outscaleFileStorageService; - default -> throw new ProviderNotFoundException(); - }; - } - - @Override - public void delete(StorageFileToDelete storageFileToDelete) { - if (storageFileToDelete == null) { - return; - } - if (storageFileToDelete.getProviders() == null) { - storageFileToDeleteRepository.delete(storageFileToDelete); - return; - } - for (String provider : storageFileToDelete.getProviders()) { - getStorageService(ObjectStorageProvider.valueOf(provider)).delete(storageFileToDelete.getPath()); - } - storageFileToDeleteRepository.delete(storageFileToDelete); - } - - -} \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageToDeleteServiceMock.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageToDeleteServiceMock.java deleted file mode 100644 index 93f80618d..000000000 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/FileStorageToDeleteServiceMock.java +++ /dev/null @@ -1,20 +0,0 @@ -package fr.dossierfacile.common.service; - -import fr.dossierfacile.common.entity.StorageFileToDelete; -import fr.dossierfacile.common.service.interfaces.FileStorageToDeleteService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Service; - -@Service -@Slf4j -@Profile("mockOvh") -public class FileStorageToDeleteServiceMock implements FileStorageToDeleteService { - public FileStorageToDeleteServiceMock() { - } - - @Override - public void delete(StorageFileToDelete storageFileToDelete) { - log.warn("Mock version - do nothing"); - } -} \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/MockStorage.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LocalMockStorage.java similarity index 54% rename from dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/MockStorage.java rename to dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LocalMockStorage.java index 5d6e67cfb..ae2c72f1d 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/MockStorage.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LocalMockStorage.java @@ -2,75 +2,70 @@ import fr.dossierfacile.common.entity.EncryptionKey; import fr.dossierfacile.common.entity.ObjectStorageProvider; -import fr.dossierfacile.common.entity.StorageFile; import fr.dossierfacile.common.exceptions.RetryableOperationException; import fr.dossierfacile.common.exceptions.UnsupportedKeyException; -import fr.dossierfacile.common.repository.StorageFileRepository; -import fr.dossierfacile.common.service.interfaces.FileStorageService; +import fr.dossierfacile.common.service.interfaces.FileStorageProviderService; +import jakarta.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import javax.crypto.Cipher; import javax.crypto.CipherInputStream; import javax.crypto.NoSuchPaddingException; import javax.crypto.spec.GCMParameterSpec; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; +import java.io.*; +import java.nio.file.*; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; -import java.security.Key; import java.security.NoSuchAlgorithmException; -import java.util.Collections; -import java.util.UUID; +import java.util.List; import static java.lang.String.format; @Slf4j @Service @Profile("mockOvh") -public class MockStorage implements FileStorageService { +public class LocalMockStorage implements FileStorageProviderService { private final String filePath; - private final StorageFileRepository storageFileRepository; - public MockStorage(@Value("${mock.storage.path:./mockstorage/}") String filePath, StorageFileRepository storageFileRepository) { + public LocalMockStorage(@Value("${mock.storage.path:./mockstorage/}") String filePath) { this.filePath = filePath; - this.storageFileRepository = storageFileRepository; new File(filePath).mkdirs(); } @Override - public void delete(StorageFile storageFile) { + public ObjectStorageProvider getProvider() { + return ObjectStorageProvider.LOCAL; + } + + @Override + @Async + public void delete(String path) { try { - Files.delete(Path.of(filePath + storageFile.getPath())); + Files.delete(Path.of(filePath + path)); } catch (NoSuchFileException e) { - log.error(format("File %s does not exist", filePath + storageFile.getPath()), e); + log.error(format("File %s does not exist", filePath + path), e); } catch (DirectoryNotEmptyException e) { - log.error(format("File %s is a non empty directory", filePath + storageFile.getPath()), e); + log.error(format("File %s is a non empty directory", filePath + path), e); } catch (IOException e) { - log.error(format("File %s cannot be deleted", filePath + storageFile.getPath()), e); + log.error(format("File %s cannot be deleted", filePath + path), e); } } - private InputStream download(String filename, EncryptionKey key) throws IOException { - InputStream in = Files.newInputStream(Path.of(filePath + filename)); + @Override + public InputStream download(String path, EncryptionKey key) throws IOException { + InputStream in = Files.newInputStream(Path.of(filePath + path)); if (key != null) { - if (key.getVersion() != 1){ + if (key.getVersion() != 1) { throw new UnsupportedKeyException("Unsupported key version " + key.getVersion()); } try { - byte[] iv = DigestUtils.md5(filename); + byte[] iv = DigestUtils.md5(path); GCMParameterSpec gcmParamSpec = new GCMParameterSpec(128, iv); Cipher aes = Cipher.getInstance("AES/GCM/NoPadding"); aes.init(Cipher.DECRYPT_MODE, key, gcmParamSpec); @@ -85,11 +80,7 @@ private InputStream download(String filename, EncryptionKey key) throws IOExcept } @Override - public InputStream download(StorageFile storageFile) throws IOException { - return download(storageFile.getPath(), storageFile.getEncryptionKey()); - } - - public void upload(String ovhPath, InputStream inputStream, Key key, String contentType) throws IOException { + public void upload(String ovhPath, InputStream inputStream, EncryptionKey key, String contentType) throws RetryableOperationException, IOException { if (key != null) { try { byte[] iv = DigestUtils.md5(ovhPath); @@ -116,34 +107,17 @@ public void upload(String ovhPath, InputStream inputStream, Key key, String cont } @Override - public StorageFile upload(InputStream inputStream, StorageFile storageFile) throws IOException { - try { - return uploadToProvider(inputStream, storageFile, ObjectStorageProvider.OVH); - } catch (RetryableOperationException e) { - return null; - } - } - - @Override - public StorageFile uploadToProvider(InputStream inputStream, StorageFile storageFile, ObjectStorageProvider provider) throws RetryableOperationException, IOException { - if (inputStream == null) - return null; - if (storageFile == null) { - log.warn("fallback on uploadfile"); - storageFile = StorageFile.builder() - .name("undefined") - .provider(provider) - .providers(Collections.singletonList(provider.toString())) - .build(); - } + public List listObjectNames(@Nullable String marker, int maxObjects) { - if (StringUtils.isBlank(storageFile.getPath())) { - storageFile.setPath(UUID.randomUUID().toString()); + Path directory = Paths.get(filePath); + if (Files.isDirectory(directory)) { + try { + return Files.list(directory).map(Path::toString).toList(); + } catch (IOException e) { + throw new RuntimeException(e); + } } - upload(storageFile.getPath(), inputStream, storageFile.getEncryptionKey(), storageFile.getContentType()); - - return storageFileRepository.save(storageFile); - + throw new IllegalStateException("File path" + filePath + " is not a directory"); } } \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/FileStorageService.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/FileStorageService.java index 4503f289b..18737b4ec 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/FileStorageService.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/FileStorageService.java @@ -9,8 +9,11 @@ public interface FileStorageService { + /** + * Soft delete + */ void delete(StorageFile storageFile); - + void hardDelete(StorageFile storageFile); /** * Get the downloaded file's inputStream. * If {@code key} is null then the inputStream is directly returned without decrypt operation. diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/FileStorageToDeleteService.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/FileStorageToDeleteService.java deleted file mode 100644 index 05ef3174e..000000000 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/FileStorageToDeleteService.java +++ /dev/null @@ -1,9 +0,0 @@ -package fr.dossierfacile.common.service.interfaces; - -import fr.dossierfacile.common.entity.StorageFileToDelete; - -public interface FileStorageToDeleteService { - - void delete(StorageFileToDelete storageFileToDelete); - -} \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/resources/db/changelog/databaseChangeLog.xml b/dossierfacile-common-library/src/main/resources/db/changelog/databaseChangeLog.xml index 384df085d..f12c51714 100644 --- a/dossierfacile-common-library/src/main/resources/db/changelog/databaseChangeLog.xml +++ b/dossierfacile-common-library/src/main/resources/db/changelog/databaseChangeLog.xml @@ -139,5 +139,6 @@ + diff --git a/dossierfacile-common-library/src/main/resources/db/migration/202404240000-add-retry-storage-file.xml b/dossierfacile-common-library/src/main/resources/db/migration/202404240000-add-retry-storage-file.xml new file mode 100644 index 000000000..cf2c70a72 --- /dev/null +++ b/dossierfacile-common-library/src/main/resources/db/migration/202404240000-add-retry-storage-file.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + INSERT INTO storage_file (id, created_by, created_date, last_modified_by, last_modified_date, name, path, label, content_type, size, md5, provider, encryption_key_id, providers, status) + SELECT id, created_by, created_date, last_modified_by, last_modified_date, path, path, path, null, 1, null, 'OVH', 1, providers, 'TO_DELETE' + FROM storage_file_to_delete; + + + + + + \ No newline at end of file diff --git a/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/ApartmentSharingPdfDossierFileGenerationServiceImpl.java b/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/ApartmentSharingPdfDossierFileGenerationServiceImpl.java index 105f52d2a..c4061a7ba 100644 --- a/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/ApartmentSharingPdfDossierFileGenerationServiceImpl.java +++ b/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/ApartmentSharingPdfDossierFileGenerationServiceImpl.java @@ -6,6 +6,7 @@ import fr.dossierfacile.common.enums.FileStatus; import fr.dossierfacile.common.repository.StorageFileRepository; import fr.dossierfacile.common.service.interfaces.ApartmentSharingCommonService; +import fr.dossierfacile.common.service.interfaces.FileStorageService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -19,7 +20,7 @@ @Transactional public class ApartmentSharingPdfDossierFileGenerationServiceImpl implements ApartmentSharingPdfDossierFileGenerationService { private final ApartmentSharingCommonService apartmentSharingCommonService; - private final StorageFileRepository storageFileRepository; + private final FileStorageService fileStorageService; @Override public void complete(Long id, StorageFile file) { @@ -35,10 +36,8 @@ public void fail(Long apartmentSharingId, StorageFile file) { apartmentSharing.setDossierPdfDocumentStatus(FileStatus.FAILED); apartmentSharing.setPdfDossierFile(null); apartmentSharingCommonService.save(apartmentSharing); + fileStorageService.delete(file); - if (file != null) { - storageFileRepository.delete(file); - } } @Override diff --git a/dossierfacile-pdf-generator/src/main/resources/application.properties b/dossierfacile-pdf-generator/src/main/resources/application.properties index 5e65739bf..d86832992 100644 --- a/dossierfacile-pdf-generator/src/main/resources/application.properties +++ b/dossierfacile-pdf-generator/src/main/resources/application.properties @@ -30,7 +30,7 @@ threeds.access.key.secret= threeds.s3.service.endpoint= threeds.s3.service.region= threeds.s3.bucket= -storage.provider.list=THREEDS_OUTSCALE,OVH +storage.provider.list=OVH,THREEDS_OUTSCALE # Logging logging.config=classpath:logback-spring-delayed.xml diff --git a/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/service/MockStorageServiceTest.java b/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/service/MockStorageServiceTest.java deleted file mode 100644 index d0088e650..000000000 --- a/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/service/MockStorageServiceTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package fr.dossierfacile.api.pdfgenerator.service; - -import fr.dossierfacile.common.entity.EncryptionKey; -import fr.dossierfacile.common.entity.StorageFile; -import fr.dossierfacile.common.repository.EncryptionKeyRepository; -import fr.dossierfacile.common.repository.StorageFileRepository; -import fr.dossierfacile.common.service.EncryptionKeyServiceImpl; -import fr.dossierfacile.common.service.MockStorage; -import org.apache.pdfbox.io.IOUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; - -@ExtendWith(MockitoExtension.class) -class MockStorageServiceTest { - - @Mock - EncryptionKeyRepository repository; - @Mock - StorageFileRepository storageFileRepository; - @InjectMocks - EncryptionKeyServiceImpl encryptionKeyService; - - MockStorage fileStorageService; - - @BeforeEach - void init() { - fileStorageService = new MockStorage("./mockstorage/", storageFileRepository); - Mockito.when(repository.save(Mockito.any())).thenAnswer(i -> i.getArguments()[0]); - Mockito.when(storageFileRepository.save(Mockito.any())).thenAnswer(i -> i.getArguments()[0]); - } - - @Test - void check_upload_download_with_mockstorage() throws IOException { - EncryptionKey key = encryptionKeyService.getCurrentKey(); - - StorageFile file = fileStorageService.upload(new ByteArrayInputStream(new byte[]{1, 2, 3}), StorageFile.builder().name("test").encryptionKey(key).build()); - InputStream result = fileStorageService.download(file); - Assertions.assertArrayEquals(new byte[]{1, 2, 3}, IOUtils.toByteArray(result)); - } -} \ No newline at end of file diff --git a/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/service/PdfGeneratorServiceImplTest.java b/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/service/PdfGeneratorServiceImplTest.java index 9aace56ae..9f7a375fb 100644 --- a/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/service/PdfGeneratorServiceImplTest.java +++ b/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/service/PdfGeneratorServiceImplTest.java @@ -15,6 +15,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; import java.io.File; import java.io.IOException; @@ -35,7 +36,6 @@ class PdfGeneratorServiceImplTest { @MockBean StorageFileRepository storageFileRepository; - @Value("${mock.storage.path}") private String filePath; diff --git a/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/util/parameterresolvers/TenantResolver.java b/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/util/parameterresolvers/TenantResolver.java index f5e569857..7ee20ad14 100644 --- a/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/util/parameterresolvers/TenantResolver.java +++ b/dossierfacile-pdf-generator/src/test/java/fr/dossierfacile/api/pdfgenerator/util/parameterresolvers/TenantResolver.java @@ -1,6 +1,7 @@ package fr.dossierfacile.api.pdfgenerator.util.parameterresolvers; import fr.dossierfacile.common.entity.Document; +import fr.dossierfacile.common.entity.ObjectStorageProvider; import fr.dossierfacile.common.entity.StorageFile; import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.enums.DocumentCategory; @@ -14,6 +15,7 @@ import org.junit.jupiter.api.extension.support.TypeBasedParameterResolver; import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class TenantResolver extends TypeBasedParameterResolver { @@ -48,13 +50,21 @@ static Tenant buildTenant() { return tenant; } + private static StorageFile buildStorageFile() { + return StorageFile.builder() + .path("CNI.pdf") + .provider(ObjectStorageProvider.LOCAL) + .providers(Collections.singletonList(ObjectStorageProvider.LOCAL.name())) + .build(); + } + private static List buildDocuments(Tenant tenant) { Document professional = new Document(); professional.setDocumentCategory(DocumentCategory.PROFESSIONAL); professional.setDocumentSubCategory(DocumentSubCategory.CDI); professional.setDocumentStatus(DocumentStatus.VALIDATED); professional.setTenant(tenant); - professional.setWatermarkFile(StorageFile.builder().path("CNI.pdf").build()); + professional.setWatermarkFile(buildStorageFile()); Document financial = new Document(); financial.setDocumentCategory(DocumentCategory.FINANCIAL); @@ -62,28 +72,28 @@ private static List buildDocuments(Tenant tenant) { financial.setDocumentStatus(DocumentStatus.VALIDATED); financial.setTenant(tenant); financial.setMonthlySum(3000); - financial.setWatermarkFile(StorageFile.builder().path("CNI.pdf").build()); + financial.setWatermarkFile(buildStorageFile()); Document tax = new Document(); tax.setDocumentCategory(DocumentCategory.TAX); tax.setDocumentSubCategory(DocumentSubCategory.LESS_THAN_YEAR); tax.setDocumentStatus(DocumentStatus.VALIDATED); tax.setTenant(tenant); - tax.setWatermarkFile(StorageFile.builder().path("CNI.pdf").build()); + tax.setWatermarkFile(buildStorageFile()); Document identification = new Document(); identification.setDocumentCategory(DocumentCategory.IDENTIFICATION); identification.setDocumentSubCategory(DocumentSubCategory.FRENCH_PASSPORT); identification.setDocumentStatus(DocumentStatus.VALIDATED); identification.setTenant(tenant); - identification.setWatermarkFile(StorageFile.builder().path("CNI.pdf").build()); + identification.setWatermarkFile(buildStorageFile()); Document residency = new Document(); residency.setDocumentCategory(DocumentCategory.RESIDENCY); residency.setDocumentSubCategory(DocumentSubCategory.TENANT); residency.setDocumentStatus(DocumentStatus.VALIDATED); residency.setTenant(tenant); - residency.setWatermarkFile(StorageFile.builder().path("CNI.pdf").build()); + residency.setWatermarkFile(buildStorageFile()); return List.of(professional, financial, tax, identification, residency); } diff --git a/dossierfacile-pdf-generator/src/test/resources/application.properties b/dossierfacile-pdf-generator/src/test/resources/application.properties index b38a864c7..26f38b457 100644 --- a/dossierfacile-pdf-generator/src/test/resources/application.properties +++ b/dossierfacile-pdf-generator/src/test/resources/application.properties @@ -4,4 +4,4 @@ threeds.access.key.secret= threeds.s3.service.endpoint=https://oos.cloudgouv-eu-west-1.outscale.com threeds.s3.service.region= threeds.s3.bucket= -storage.provider.list=THREEDS_OUTSCALE,OVH +storage.provider.list=LOCAL diff --git a/dossierfacile-process-file/src/main/resources/application.properties b/dossierfacile-process-file/src/main/resources/application.properties index 99f02073b..0c63c3f55 100644 --- a/dossierfacile-process-file/src/main/resources/application.properties +++ b/dossierfacile-process-file/src/main/resources/application.properties @@ -14,6 +14,9 @@ spring.datasource.url= spring.datasource.username= spring.datasource.password= +# Storage configuration +storage.provider.list=OVH,THREEDS_OUTSCALE + #OVH Storage Configuration ovh.auth.url= ovh.username= diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/LoggingContext.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/LoggingContext.java index 6110fe872..8b5c50dda 100644 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/LoggingContext.java +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/LoggingContext.java @@ -1,6 +1,5 @@ package fr.dossierfacile.scheduler; -import fr.dossierfacile.common.entity.StorageFile; import fr.dossierfacile.scheduler.tasks.TaskName; import lombok.extern.slf4j.Slf4j; import org.slf4j.MDC; @@ -8,7 +7,7 @@ @Slf4j public class LoggingContext { - private static final String STORAGE_FILE = "storage_file_id"; + public static final String STORAGE_FILE = "storage_file_id"; private static final String TASK_NAME = "task"; public static void startTask(TaskName taskName) { @@ -21,10 +20,12 @@ public static void endTask() { MDC.clear(); } - public static void setStorageFile(StorageFile file) { - if (file.getId() != null) { - MDC.put(STORAGE_FILE, file.getId().toString()); - } + public static void put(String key, Object value) { + MDC.put(key, String.valueOf(value)); + } + + public static void remove(String key) { + MDC.remove(key); } public static void clear() { diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/TaskName.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/TaskName.java index d89694e75..1bf87a9af 100644 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/TaskName.java +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/TaskName.java @@ -7,7 +7,9 @@ public enum TaskName { TENANT_ARCHIVING, TENANT_DELETION, STORAGE_FILES_BACKUP, + STORAGE_FILES_BACKUP_RETRY, STORAGE_FILES_DELETION, + STORAGE_FILES_DELETION_RETRY, PDF_GENERATION, DELETE_FAILED_DOCUMENT; diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/BackupFilesTask.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/BackupFilesTask.java index f1a260c21..00552d6b0 100644 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/BackupFilesTask.java +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/BackupFilesTask.java @@ -1,5 +1,6 @@ package fr.dossierfacile.scheduler.tasks.storagesynchronization; +import fr.dossierfacile.common.config.DynamicProviderConfig; import fr.dossierfacile.common.entity.ObjectStorageProvider; import fr.dossierfacile.common.entity.StorageFile; import fr.dossierfacile.common.repository.StorageFileRepository; @@ -15,7 +16,9 @@ import java.io.InputStream; import java.util.List; +import static fr.dossierfacile.scheduler.LoggingContext.STORAGE_FILE; import static fr.dossierfacile.scheduler.tasks.TaskName.STORAGE_FILES_BACKUP; +import static fr.dossierfacile.scheduler.tasks.TaskName.STORAGE_FILES_BACKUP_RETRY; @Slf4j @Service @@ -23,34 +26,44 @@ public class BackupFilesTask { private final StorageFileRepository storageFileRepository; private final FileStorageService fileStorageService; + private final DynamicProviderConfig dynamicProviderConfig; + + private static boolean isNotPresentOnProvider(StorageFile storageFile, ObjectStorageProvider objectStorageProvider) { + return !storageFile.getProviders().contains(objectStorageProvider.name()); + } @Scheduled(fixedDelayString = "${scheduled.process.storage.backup.delay.ms}", initialDelayString = "${scheduled.process.storage.backup.delay.ms}") public void scheduleBackupTask() { LoggingContext.startTask(STORAGE_FILES_BACKUP); Pageable limit = PageRequest.of(0, 100); - List storageFiles = storageFileRepository.findAllWithOneProvider(limit); - storageFiles.parallelStream().forEach(storageFile -> { - LoggingContext.setStorageFile(storageFile); - for (ObjectStorageProvider objectStorageProvider : ObjectStorageProvider.values()) { - if (isNotPresentOnProvider(storageFile, objectStorageProvider)) { - uploadFileToProvider(storageFile, objectStorageProvider); - } - } - LoggingContext.clear(); - }); + List storageFiles = storageFileRepository.findAllWithOneProviderAndReady(limit); + synchronizeFile(storageFiles); LoggingContext.endTask(); } - private static boolean isNotPresentOnProvider(StorageFile storageFile, ObjectStorageProvider objectStorageProvider) { - return !storageFile.getProviders().contains(objectStorageProvider.name()); + @Scheduled(fixedDelayString = "${scheduled.process.storage.backup.retry.failed.copy.delay.minutes}", initialDelayString = "${scheduled.process.storage.backup.retry.failed.copy.delay.minutes}") + public void retryFailedCopy() { + LoggingContext.startTask(STORAGE_FILES_BACKUP_RETRY); + Pageable limit = PageRequest.of(0, 100); + List storageFiles = storageFileRepository.findAllWithOneProviderAndCopyFailed(limit); + synchronizeFile(storageFiles); + LoggingContext.endTask(); } - private void uploadFileToProvider(StorageFile storageFile, ObjectStorageProvider objectStorageProvider) { - try (InputStream is = fileStorageService.download(storageFile)) { - fileStorageService.uploadToProvider(is, storageFile, objectStorageProvider); - } catch (Exception e) { - log.error("Upload to {} failed", objectStorageProvider); - } + private void synchronizeFile(List storageFiles) { + storageFiles.forEach(storageFile -> { + LoggingContext.put(STORAGE_FILE, storageFile.getId()); + for (ObjectStorageProvider objectStorageProvider : dynamicProviderConfig.getProviders()) { + if (isNotPresentOnProvider(storageFile, objectStorageProvider)) { + try (InputStream is = fileStorageService.download(storageFile)) { + fileStorageService.uploadToProvider(is, storageFile, objectStorageProvider); + } catch (Exception e) { + log.error("Failed copy for {} to {}", storageFile.getId(), objectStorageProvider); + } + } + } + LoggingContext.remove(STORAGE_FILE); + }); } } diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/DeleteFilesTask.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/DeleteFilesTask.java index ff475847d..63cc78a4a 100644 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/DeleteFilesTask.java +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/DeleteFilesTask.java @@ -1,8 +1,9 @@ package fr.dossierfacile.scheduler.tasks.storagesynchronization; -import fr.dossierfacile.common.entity.StorageFileToDelete; -import fr.dossierfacile.common.repository.StorageFileToDeleteRepository; -import fr.dossierfacile.common.service.interfaces.FileStorageToDeleteService; +import fr.dossierfacile.common.entity.StorageFile; +import fr.dossierfacile.common.enums.FileStorageStatus; +import fr.dossierfacile.common.repository.StorageFileRepository; +import fr.dossierfacile.common.service.interfaces.FileStorageService; import fr.dossierfacile.scheduler.LoggingContext; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -12,21 +13,32 @@ import java.util.List; import static fr.dossierfacile.scheduler.tasks.TaskName.STORAGE_FILES_DELETION; +import static fr.dossierfacile.scheduler.tasks.TaskName.STORAGE_FILES_DELETION_RETRY; @Slf4j @Service @RequiredArgsConstructor public class DeleteFilesTask { - private final StorageFileToDeleteRepository storageFileToDeleteRepository; - private final FileStorageToDeleteService fileStorageToDeleteService; + private final StorageFileRepository storageFileRepository; + private final FileStorageService fileStorageService; @Scheduled(fixedDelayString = "${scheduled.process.storage.delete.delay.ms}", initialDelayString = "${scheduled.process.storage.delete.delay.ms}") public void deleteFileInProviderTask() { LoggingContext.startTask(STORAGE_FILES_DELETION); - List storageFileToDeleteList = storageFileToDeleteRepository.findAll(); - for (StorageFileToDelete storageFileToDelete : storageFileToDeleteList) { - fileStorageToDeleteService.delete(storageFileToDelete); + List storageFileToDeleteList = storageFileRepository.findAllByStatus(FileStorageStatus.TO_DELETE); + for (StorageFile storageFileToDelete : storageFileToDeleteList) { + fileStorageService.hardDelete(storageFileToDelete); + } + LoggingContext.endTask(); + } + + @Scheduled(fixedDelayString = "${scheduled.process.storage.delete.retry.failed.copy.delay.minutes}", initialDelayString = "${scheduled.process.storage.delete.delay.ms}") + public void retryDeleteFileInProviderTask() { + LoggingContext.startTask(STORAGE_FILES_DELETION_RETRY); + List storageFileToDeleteList = storageFileRepository.findAllByStatus(FileStorageStatus.DELETE_FAILED); + for (StorageFile storageFileToDelete : storageFileToDeleteList) { + fileStorageService.hardDelete(storageFileToDelete); } LoggingContext.endTask(); } diff --git a/dossierfacile-task-scheduler/src/main/resources/application.properties b/dossierfacile-task-scheduler/src/main/resources/application.properties index ecc4c06ba..8e8a1fd02 100644 --- a/dossierfacile-task-scheduler/src/main/resources/application.properties +++ b/dossierfacile-task-scheduler/src/main/resources/application.properties @@ -17,7 +17,7 @@ ovh.container= server.port=8089 app.username= app.password= -storage.provider.list=THREEDS_OUTSCALE,OVH +storage.provider.list=OVH,THREEDS_OUTSCALE spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect # keycloak @@ -45,7 +45,9 @@ cron.account-deletion= cron.process.pdf.generation.failed=0 30 1,7,12,19 * * * cron.delete.document.with.failed.pdf=0 0 6,22 * * * scheduled.process.storage.backup.delay.ms=10000 +scheduled.process.storage.backup.retry.failed.copy.delay.minutes=5 scheduled.process.storage.delete.delay.ms=10000 +scheduled.process.storage.delete.retry.failed.copy.delay.minutes=5 garbage-collection.seconds-between-iterations=60 document.pdf.failed.delay.before.delete.hours=480