From 52cd070e67cdb5cb470700072c10f410326d00d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:48:08 +0100 Subject: [PATCH 01/19] chore(deps): bump ch.qos.logback:logback-core from 1.2.12 to 1.2.13 in /dossierfacile-task-scheduler (#668) Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.2.12 to 1.2.13. - [Commits](https://github.com/qos-ch/logback/compare/v_1.2.12...v_1.2.13) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dossierfacile-task-scheduler/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dossierfacile-task-scheduler/pom.xml b/dossierfacile-task-scheduler/pom.xml index 141084021..c2ffe7186 100644 --- a/dossierfacile-task-scheduler/pom.xml +++ b/dossierfacile-task-scheduler/pom.xml @@ -92,7 +92,7 @@ ch.qos.logback logback-core - 1.2.12 + 1.2.13 net.logstash.logback From 15d3ed882ed041e5b865f2af8c1c086ffb9a8035 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:48:27 +0100 Subject: [PATCH 02/19] chore(deps): bump ch.qos.logback:logback-core from 1.2.12 to 1.2.13 in /dossierfacile-api-tenant (#666) Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.2.12 to 1.2.13. - [Commits](https://github.com/qos-ch/logback/compare/v_1.2.12...v_1.2.13) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dossierfacile-api-tenant/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dossierfacile-api-tenant/pom.xml b/dossierfacile-api-tenant/pom.xml index 1f7344fe1..cc1eaa982 100644 --- a/dossierfacile-api-tenant/pom.xml +++ b/dossierfacile-api-tenant/pom.xml @@ -46,7 +46,7 @@ ch.qos.logback logback-core - 1.2.12 + 1.2.13 org.apache.pdfbox From b10f51eed20f8a7ca963d0605f940e0a050874b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:48:34 +0100 Subject: [PATCH 03/19] chore(deps): bump ch.qos.logback:logback-core from 1.2.12 to 1.2.13 in /dossierfacile-pdf-generator (#667) Bumps [ch.qos.logback:logback-core](https://github.com/qos-ch/logback) from 1.2.12 to 1.2.13. - [Commits](https://github.com/qos-ch/logback/compare/v_1.2.12...v_1.2.13) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- dossierfacile-pdf-generator/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dossierfacile-pdf-generator/pom.xml b/dossierfacile-pdf-generator/pom.xml index 604a84053..f5f6f37d5 100644 --- a/dossierfacile-pdf-generator/pom.xml +++ b/dossierfacile-pdf-generator/pom.xml @@ -47,7 +47,7 @@ ch.qos.logback logback-core - 1.2.12 + 1.2.13 org.apache.pdfbox From ff4a47a5f107adbd813bb9c8fcd2758c0f38bec5 Mon Sep 17 00:00:00 2001 From: Juliette de Rancourt Date: Tue, 19 Dec 2023 18:05:26 +0100 Subject: [PATCH 04/19] chore(ci): Upgrade JDK versions in github workflows --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 51219f771..c56d0bad0 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -45,7 +45,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: '18' + java-version: '21' # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dacaa87cf..6652cd296 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 20 + - name: Set up JDK 21 uses: actions/setup-java@v3 with: - java-version: '20' + java-version: '21' distribution: 'temurin' cache: maven - name: Build & test From a3f672bc017201d8f7a875901c214000955c7127 Mon Sep 17 00:00:00 2001 From: Tristan Robert Date: Tue, 19 Dec 2023 18:13:11 +0100 Subject: [PATCH 05/19] [Snyk] Upgrade com.fasterxml.jackson.core:jackson-databind from 2.13.4.2 to 2.15.2 (#547) Snyk has created this PR to upgrade com.fasterxml.jackson.core:jackson-databind from 2.13.4.2 to 2.15.2. See this package in Maven Repository: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind/ See this project in Snyk: https://app.snyk.io/org/mtes-mct/project/e26ce31f-9859-4380-80ea-9c66319589e4?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot --- dossierfacile-process-file/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dossierfacile-process-file/pom.xml b/dossierfacile-process-file/pom.xml index 8bb883291..cd7b4108e 100644 --- a/dossierfacile-process-file/pom.xml +++ b/dossierfacile-process-file/pom.xml @@ -75,7 +75,7 @@ com.fasterxml.jackson.core jackson-databind - 2.13.4.2 + 2.15.2 com.fasterxml.jackson.core From ae6d94bc901eb6e6d6153d605a9a70febbfeefc8 Mon Sep 17 00:00:00 2001 From: Tristan Robert Date: Tue, 19 Dec 2023 18:13:45 +0100 Subject: [PATCH 06/19] [Snyk] Upgrade com.fasterxml.jackson.core:jackson-core from 2.13.4 to 2.15.2 (#546) Snyk has created this PR to upgrade com.fasterxml.jackson.core:jackson-core from 2.13.4 to 2.15.2. See this package in Maven Repository: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core/ See this project in Snyk: https://app.snyk.io/org/mtes-mct/project/e26ce31f-9859-4380-80ea-9c66319589e4?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot --- dossierfacile-process-file/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dossierfacile-process-file/pom.xml b/dossierfacile-process-file/pom.xml index cd7b4108e..cfb5cd9ba 100644 --- a/dossierfacile-process-file/pom.xml +++ b/dossierfacile-process-file/pom.xml @@ -80,7 +80,7 @@ com.fasterxml.jackson.core jackson-core - 2.13.4 + 2.15.2 com.fasterxml.jackson.core From 22a088eb928158436fb781363304031585a346aa Mon Sep 17 00:00:00 2001 From: Tristan Robert Date: Tue, 19 Dec 2023 18:14:36 +0100 Subject: [PATCH 07/19] [Snyk] Upgrade com.vladmihalcea:hibernate-types-52 from 2.13.0 to 2.21.1 (#550) Snyk has created this PR to upgrade com.vladmihalcea:hibernate-types-52 from 2.13.0 to 2.21.1. See this package in Maven Repository: https://mvnrepository.com/artifact/com.vladmihalcea/hibernate-types-52/ See this project in Snyk: https://app.snyk.io/org/mtes-mct/project/76e72868-d6f4-441a-a731-1717ab12c313?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot From 270d9e0815299201e3a6d034196ee92f6fc49dd0 Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 19 Dec 2023 18:19:03 +0100 Subject: [PATCH 08/19] chore: upgrade names rules on income tax --- .../IncomeTaxRulesValidationService.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java index 4425ac7d1..f5e5d390b 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java @@ -21,6 +21,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; +import java.text.Normalizer; import java.time.LocalDate; import java.util.ArrayList; import java.util.LinkedList; @@ -49,6 +50,14 @@ private TaxIncomeMainFile fromQR(BarCodeFileAnalysis barCodeFileAnalysis) { .referenceAvis(dataWithLabel.get(TwoDDocDataType.ID_44.getLabel())).build(); } + private static String normalizeName(String name) { + if (name == null) + return null; + String normalized = Normalizer.normalize(name, Normalizer.Form.NFD); + return normalized.replace('-', ' ') + .replaceAll("[\\p{InCombiningDiacriticalMarks}]", "").toUpperCase(); + } + @Override @Transactional public DocumentAnalysisReport process(Document document) { @@ -158,11 +167,11 @@ public DocumentAnalysisReport process(Document document) { if (analysis.getDocumentType() == BarCodeDocumentType.TAX_ASSESSMENT) { TaxIncomeMainFile qrDocument = fromQR(dfFile.getFileAnalysis()); - if (!(qrDocument.getDeclarant1Nom().contains(firstName) - && qrDocument.getDeclarant1Nom().contains(lastName) + if (!(normalizeName(qrDocument.getDeclarant1Nom()).contains(normalizeName(firstName)) + && normalizeName(qrDocument.getDeclarant1Nom()).contains(normalizeName(lastName)) || (qrDocument.getDeclarant2Nom() != null && - qrDocument.getDeclarant2Nom().contains(firstName) - && qrDocument.getDeclarant2Nom().contains(lastName) + normalizeName(qrDocument.getDeclarant2Nom()).contains(normalizeName(firstName)) + && normalizeName(qrDocument.getDeclarant2Nom()).contains(normalizeName(lastName)) ))) { log.error("Le nom/prenom ne correpond pas à l'uilitsation tenantId:" + document.getTenant().getId() + " firstname: " + firstName); brokenRules.add(DocumentBrokenRule.builder() From d17163137c147e68c1afae906f7abebec70e7151 Mon Sep 17 00:00:00 2001 From: Juliette de Rancourt Date: Tue, 19 Dec 2023 15:38:38 +0100 Subject: [PATCH 09/19] feat(tenant): Log whether document edition is adding/deleting files --- .../api/front/register/AbstractDocumentSaveStep.java | 3 ++- .../api/front/service/DocumentServiceImpl.java | 5 ++++- .../api/front/service/FileServiceImpl.java | 5 ++--- .../common/model/EditedDocumentModel.java | 6 +++--- .../fr/dossierfacile/common/model/EditionType.java | 6 ++++++ .../dossierfacile/common/service/LogServiceImpl.java | 9 +++++---- .../common/service/interfaces/LogService.java | 3 ++- .../common/service/LogServiceImplTest.java | 10 +++++----- 8 files changed, 29 insertions(+), 18 deletions(-) create mode 100644 dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditionType.java diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/AbstractDocumentSaveStep.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/AbstractDocumentSaveStep.java index 85355f1a6..4934ade44 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/AbstractDocumentSaveStep.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/AbstractDocumentSaveStep.java @@ -10,6 +10,7 @@ import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.enums.PartnerCallBackType; import fr.dossierfacile.common.enums.TenantFileStatus; +import fr.dossierfacile.common.model.EditionType; import fr.dossierfacile.common.repository.TenantCommonRepository; import fr.dossierfacile.common.service.interfaces.LogService; import fr.dossierfacile.common.service.interfaces.PartnerCallBackService; @@ -46,7 +47,7 @@ public TenantModel saveStep(Tenant tenant, T documentForm) { } Document document = saveDocument(tenant, documentForm); - logService.saveDocumentEditedLog(document, tenant); + logService.saveDocumentEditedLog(document, tenant, EditionType.ADD); documentService.markDocumentAsEdited(document); TransactionalUtil.afterCommit(() -> { diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java index 8f1477737..2031b89de 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java @@ -9,7 +9,6 @@ import fr.dossierfacile.api.front.service.interfaces.TenantStatusService; import fr.dossierfacile.api.front.util.TransactionalUtil; import fr.dossierfacile.common.entity.Document; -import fr.dossierfacile.common.entity.DocumentAnalysisReport; import fr.dossierfacile.common.entity.File; import fr.dossierfacile.common.entity.Person; import fr.dossierfacile.common.entity.StorageFile; @@ -17,9 +16,11 @@ import fr.dossierfacile.common.enums.DocumentCategory; import fr.dossierfacile.common.enums.DocumentStatus; import fr.dossierfacile.common.enums.TenantFileStatus; +import fr.dossierfacile.common.model.EditionType; import fr.dossierfacile.common.repository.DocumentAnalysisReportRepository; import fr.dossierfacile.common.repository.StorageFileRepository; import fr.dossierfacile.common.service.interfaces.DocumentHelperService; +import fr.dossierfacile.common.service.interfaces.LogService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -46,6 +47,7 @@ public class DocumentServiceImpl implements DocumentService { private final ApartmentSharingService apartmentSharingService; private final DocumentHelperService documentHelperService; private final MinifyFileProducer minifyFileProducer; + private final LogService logService; private final Producer producer; @Override @@ -98,6 +100,7 @@ public void delete(Long documentId, Tenant referenceTenant) { Document document = documentRepository.findByIdForApartmentSharing(documentId, referenceTenant.getApartmentSharing().getId()) .orElseThrow(() -> new DocumentNotFoundException(documentId)); delete(document); + logService.saveDocumentEditedLog(document, referenceTenant, EditionType.DELETE); } @Override diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/FileServiceImpl.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/FileServiceImpl.java index dea496020..45070dfe0 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/FileServiceImpl.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/FileServiceImpl.java @@ -10,14 +10,13 @@ import fr.dossierfacile.common.entity.File; import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.enums.DocumentStatus; +import fr.dossierfacile.common.model.EditionType; import fr.dossierfacile.common.service.interfaces.LogService; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.time.LocalDateTime; - @Service @AllArgsConstructor @Slf4j @@ -36,7 +35,7 @@ public Document delete(Long id, Tenant tenant) { document.getFiles().remove(file); fileRepository.delete(file); - logService.saveDocumentEditedLog(document, tenant); + logService.saveDocumentEditedLog(document, tenant, EditionType.DELETE); documentService.markDocumentAsEdited(document); if (document.getFiles().isEmpty()) { diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditedDocumentModel.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditedDocumentModel.java index 702be5fdb..f2f306bee 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditedDocumentModel.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditedDocumentModel.java @@ -22,19 +22,19 @@ public class EditedDocumentModel { private DocumentCategory documentCategory; private DocumentSubCategory documentSubCategory; - private Boolean noDocument; private Long tenantId; private Long guarantorId; + private EditionType editionType; - public static EditedDocumentModel from(Document document) { + public static EditedDocumentModel from(Document document, EditionType editionType) { return EditedDocumentModel.builder() .documentCategory(document.getDocumentCategory()) .documentSubCategory(document.getDocumentSubCategory()) - .noDocument(document.getNoDocument()) .tenantId(Optional.ofNullable(document.getTenant()) .map(Tenant::getId).orElse(null)) .guarantorId(Optional.ofNullable(document.getGuarantor()) .map(Guarantor::getId).orElse(null)) + .editionType(editionType) .build(); } diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditionType.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditionType.java new file mode 100644 index 000000000..b06f71398 --- /dev/null +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditionType.java @@ -0,0 +1,6 @@ +package fr.dossierfacile.common.model; + +public enum EditionType { + ADD, + DELETE +} diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java index 0cab81806..76fbdffcc 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java @@ -10,6 +10,7 @@ import fr.dossierfacile.common.enums.LogType; import fr.dossierfacile.common.mapper.DeletedTenantCommonMapper; import fr.dossierfacile.common.model.EditedDocumentModel; +import fr.dossierfacile.common.model.EditionType; import fr.dossierfacile.common.repository.LogRepository; import fr.dossierfacile.common.service.interfaces.LogService; import lombok.AllArgsConstructor; @@ -61,19 +62,19 @@ public void saveLogWithTenantData(LogType logType, Tenant tenant) { } @Override - public void saveDocumentEditedLog(Document document, Tenant editor) { + public void saveDocumentEditedLog(Document document, Tenant editor, EditionType editionType) { Log log = Log.builder() .logType(LogType.ACCOUNT_EDITED) .tenantId(editor.getId()) .creationDateTime(LocalDateTime.now()) - .logDetails(writeDocumentEditionDetails(document, editor.getId())) + .logDetails(writeDocumentEditionDetails(document, editor.getId(), editionType)) .build(); saveLog(log); } - private String writeDocumentEditionDetails(Document document, Long tenantId) { + private String writeDocumentEditionDetails(Document document, Long tenantId, EditionType editionType) { try { - return objectMapper.writeValueAsString(EditedDocumentModel.from(document)); + return objectMapper.writeValueAsString(EditedDocumentModel.from(document, editionType)); } catch (JsonProcessingException e) { log.error("Cannot write details of document edition from tenant {}", tenantId); } diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java index c88340c4c..0da744611 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java @@ -4,6 +4,7 @@ import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.entity.UserApi; import fr.dossierfacile.common.enums.LogType; +import fr.dossierfacile.common.model.EditionType; public interface LogService { @@ -11,7 +12,7 @@ public interface LogService { void saveLogWithTenantData(LogType logType, Tenant tenant); - void saveDocumentEditedLog(Document document, Tenant editor); + void saveDocumentEditedLog(Document document, Tenant editor, EditionType editionType); void savePartnerAccessRevocationLog(Tenant tenant, UserApi userApi); diff --git a/dossierfacile-common-library/src/test/java/fr/dossierfacile/common/service/LogServiceImplTest.java b/dossierfacile-common-library/src/test/java/fr/dossierfacile/common/service/LogServiceImplTest.java index 8fb4db7b9..729bced2b 100644 --- a/dossierfacile-common-library/src/test/java/fr/dossierfacile/common/service/LogServiceImplTest.java +++ b/dossierfacile-common-library/src/test/java/fr/dossierfacile/common/service/LogServiceImplTest.java @@ -9,6 +9,7 @@ import fr.dossierfacile.common.enums.DocumentCategory; import fr.dossierfacile.common.enums.DocumentSubCategory; import fr.dossierfacile.common.enums.LogType; +import fr.dossierfacile.common.model.EditionType; import fr.dossierfacile.common.repository.LogRepository; import fr.dossierfacile.common.service.interfaces.LogService; import org.junit.jupiter.api.Test; @@ -32,13 +33,13 @@ void should_save_edition_log_for_tenant_document() { .tenant(tenantWithId(2L)) .build(); - logService.saveDocumentEditedLog(document, tenantWithId(1L)); + logService.saveDocumentEditedLog(document, tenantWithId(1L), EditionType.ADD); Log savedLog = getSavedLog(); assertThat(savedLog.getLogType()).isEqualTo(LogType.ACCOUNT_EDITED); assertThat(savedLog.getTenantId()).isEqualTo(1L); assertThat(savedLog.getLogDetails()).isEqualTo(""" - {"documentCategory":"IDENTIFICATION","documentSubCategory":"FRENCH_IDENTITY_CARD","tenantId":2}"""); + {"documentCategory":"IDENTIFICATION","documentSubCategory":"FRENCH_IDENTITY_CARD","tenantId":2,"editionType":"ADD"}"""); } @Test @@ -46,17 +47,16 @@ void should_save_edition_log_for_guarantor_document() { Document document = Document.builder() .documentCategory(DocumentCategory.FINANCIAL) .documentSubCategory(DocumentSubCategory.SALARY) - .noDocument(true) .guarantor(Guarantor.builder().id(3L).build()) .build(); - logService.saveDocumentEditedLog(document, tenantWithId(2L)); + logService.saveDocumentEditedLog(document, tenantWithId(2L), EditionType.DELETE); Log savedLog = getSavedLog(); assertThat(savedLog.getLogType()).isEqualTo(LogType.ACCOUNT_EDITED); assertThat(savedLog.getTenantId()).isEqualTo(2L); assertThat(savedLog.getLogDetails()).isEqualTo(""" - {"documentCategory":"FINANCIAL","documentSubCategory":"SALARY","noDocument":true,"guarantorId":3}"""); + {"documentCategory":"FINANCIAL","documentSubCategory":"SALARY","guarantorId":3,"editionType":"DELETE"}"""); } @Test From e3245eaf7efbec41b75b140c2a668eb46ffdd95c Mon Sep 17 00:00:00 2001 From: Juliette de Rancourt Date: Wed, 20 Dec 2023 11:58:47 +0100 Subject: [PATCH 10/19] feat(tenant): Save log when application type changes --- .../register/AbstractDocumentSaveStep.java | 2 +- .../front/register/tenant/Application.java | 8 +-- .../front/service/DocumentServiceImpl.java | 2 +- .../api/front/service/FileServiceImpl.java | 2 +- .../dossierfacile/common/enums/LogType.java | 1 + .../model/log/ApplicationTypeChange.java | 18 +++++++ .../EditedDocument.java} | 8 +-- .../common/model/{ => log}/EditionType.java | 2 +- .../ApartmentSharingCommonServiceImpl.java | 13 +++-- .../common/service/LogServiceImpl.java | 51 ++++++++++++------- .../common/service/interfaces/LogService.java | 7 ++- .../common/service/LogServiceImplTest.java | 17 ++++++- 12 files changed, 96 insertions(+), 35 deletions(-) create mode 100644 dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/ApplicationTypeChange.java rename dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/{EditedDocumentModel.java => log/EditedDocument.java} (85%) rename dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/{ => log}/EditionType.java (53%) diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/AbstractDocumentSaveStep.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/AbstractDocumentSaveStep.java index 4934ade44..bfb23af54 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/AbstractDocumentSaveStep.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/AbstractDocumentSaveStep.java @@ -10,7 +10,7 @@ import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.enums.PartnerCallBackType; import fr.dossierfacile.common.enums.TenantFileStatus; -import fr.dossierfacile.common.model.EditionType; +import fr.dossierfacile.common.model.log.EditionType; import fr.dossierfacile.common.repository.TenantCommonRepository; import fr.dossierfacile.common.service.interfaces.LogService; import fr.dossierfacile.common.service.interfaces.PartnerCallBackService; diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/tenant/Application.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/tenant/Application.java index 12fa622d4..8b0b39284 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/tenant/Application.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/tenant/Application.java @@ -140,12 +140,14 @@ protected void linkEmailToTenants(Tenant tenantCreate, List } } - TenantModel saveStep(Tenant tenant, ApplicationType applicationType, List tenantToDelete, List tenantToCreate) { + TenantModel saveStep(Tenant tenant, ApplicationType newApplicationType, List tenantToDelete, List tenantToCreate) { ApartmentSharing apartmentSharing = tenant.getApartmentSharing(); - apartmentSharing.setApplicationType(applicationType); + ApplicationType currentApplicationType = apartmentSharing.getApplicationType(); + apartmentSharing.setApplicationType(newApplicationType); apartmentSharingService.resetDossierPdfGenerated(apartmentSharing); deleteCoTenants(tenantToDelete); + logService.saveApplicationTypeChangedLog(apartmentSharing.getTenants(), currentApplicationType, newApplicationType); createCoTenants(tenant, tenantToCreate, apartmentSharing); LocalDateTime now = LocalDateTime.now(); @@ -226,7 +228,7 @@ private void createCoTenants(Tenant tenantCreate, List tenants, Ap } private void deleteCoTenants(List tenantToDelete) { - tenantToDelete.stream().forEach(t -> userService.deleteAccount(t)); + tenantToDelete.forEach(userService::deleteAccount); } } diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java index 2031b89de..f878cb3e2 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/DocumentServiceImpl.java @@ -16,7 +16,7 @@ import fr.dossierfacile.common.enums.DocumentCategory; import fr.dossierfacile.common.enums.DocumentStatus; import fr.dossierfacile.common.enums.TenantFileStatus; -import fr.dossierfacile.common.model.EditionType; +import fr.dossierfacile.common.model.log.EditionType; import fr.dossierfacile.common.repository.DocumentAnalysisReportRepository; import fr.dossierfacile.common.repository.StorageFileRepository; import fr.dossierfacile.common.service.interfaces.DocumentHelperService; diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/FileServiceImpl.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/FileServiceImpl.java index 45070dfe0..a99885fe7 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/FileServiceImpl.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/FileServiceImpl.java @@ -10,7 +10,7 @@ import fr.dossierfacile.common.entity.File; import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.enums.DocumentStatus; -import fr.dossierfacile.common.model.EditionType; +import fr.dossierfacile.common.model.log.EditionType; import fr.dossierfacile.common.service.interfaces.LogService; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/LogType.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/LogType.java index 36ac66198..424a16a93 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/LogType.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/LogType.java @@ -5,6 +5,7 @@ public enum LogType { ACCOUNT_CREATED, ACCOUNT_CREATED_VIA_KC, ACCOUNT_LINK, // first connection on DF + APPLICATION_TYPE_CHANGED, ACCOUNT_COMPLETED, ACCOUNT_DENIED, ACCOUNT_EDITED, diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/ApplicationTypeChange.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/ApplicationTypeChange.java new file mode 100644 index 000000000..117bab0e2 --- /dev/null +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/ApplicationTypeChange.java @@ -0,0 +1,18 @@ +package fr.dossierfacile.common.model.log; + +import com.fasterxml.jackson.annotation.JsonInclude; +import fr.dossierfacile.common.enums.ApplicationType; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ApplicationTypeChange { + + private ApplicationType oldType; + private ApplicationType newType; + +} diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditedDocumentModel.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditedDocument.java similarity index 85% rename from dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditedDocumentModel.java rename to dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditedDocument.java index f2f306bee..b0692c53e 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditedDocumentModel.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditedDocument.java @@ -1,4 +1,4 @@ -package fr.dossierfacile.common.model; +package fr.dossierfacile.common.model.log; import com.fasterxml.jackson.annotation.JsonInclude; import fr.dossierfacile.common.entity.Document; @@ -18,7 +18,7 @@ @NoArgsConstructor @AllArgsConstructor @JsonInclude(JsonInclude.Include.NON_NULL) -public class EditedDocumentModel { +public class EditedDocument { private DocumentCategory documentCategory; private DocumentSubCategory documentSubCategory; @@ -26,8 +26,8 @@ public class EditedDocumentModel { private Long guarantorId; private EditionType editionType; - public static EditedDocumentModel from(Document document, EditionType editionType) { - return EditedDocumentModel.builder() + public static EditedDocument from(Document document, EditionType editionType) { + return EditedDocument.builder() .documentCategory(document.getDocumentCategory()) .documentSubCategory(document.getDocumentSubCategory()) .tenantId(Optional.ofNullable(document.getTenant()) diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditionType.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditionType.java similarity index 53% rename from dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditionType.java rename to dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditionType.java index b06f71398..8d1d69664 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/EditionType.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditionType.java @@ -1,4 +1,4 @@ -package fr.dossierfacile.common.model; +package fr.dossierfacile.common.model.log; public enum EditionType { ADD, diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/ApartmentSharingCommonServiceImpl.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/ApartmentSharingCommonServiceImpl.java index e764bf62f..b7733955a 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/ApartmentSharingCommonServiceImpl.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/ApartmentSharingCommonServiceImpl.java @@ -8,6 +8,7 @@ import fr.dossierfacile.common.repository.ApartmentSharingRepository; import fr.dossierfacile.common.service.interfaces.ApartmentSharingCommonService; import fr.dossierfacile.common.service.interfaces.FileStorageService; +import fr.dossierfacile.common.service.interfaces.LogService; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -20,8 +21,10 @@ @AllArgsConstructor @Slf4j public class ApartmentSharingCommonServiceImpl implements ApartmentSharingCommonService { - ApartmentSharingRepository apartmentSharingRepository; - FileStorageService fileStorageService; + + private final ApartmentSharingRepository apartmentSharingRepository; + private final FileStorageService fileStorageService; + private final LogService logService; @Override @Transactional(propagation = Propagation.SUPPORTS) @@ -43,7 +46,11 @@ public void resetDossierPdfGenerated(ApartmentSharing apartmentSharing) { @Transactional(propagation = Propagation.SUPPORTS) public void removeTenant(ApartmentSharing apartmentSharing, Tenant tenant) { apartmentSharing.getTenants().remove(tenant); - apartmentSharing.setApplicationType((apartmentSharing.getNumberOfTenants() >= 2) ? ApplicationType.GROUP : ApplicationType.ALONE); + + ApplicationType newApplicationType = (apartmentSharing.getNumberOfTenants() >= 2) ? ApplicationType.GROUP : ApplicationType.ALONE; + logService.saveApplicationTypeChangedLog(apartmentSharing.getTenants(), apartmentSharing.getApplicationType(), newApplicationType); + apartmentSharing.setApplicationType(newApplicationType); + resetDossierPdfGenerated(apartmentSharing); apartmentSharingRepository.save(apartmentSharing); } diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java index 76fbdffcc..0a49de0ef 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java @@ -7,10 +7,12 @@ import fr.dossierfacile.common.entity.Log; import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.entity.UserApi; +import fr.dossierfacile.common.enums.ApplicationType; import fr.dossierfacile.common.enums.LogType; import fr.dossierfacile.common.mapper.DeletedTenantCommonMapper; -import fr.dossierfacile.common.model.EditedDocumentModel; -import fr.dossierfacile.common.model.EditionType; +import fr.dossierfacile.common.model.log.ApplicationTypeChange; +import fr.dossierfacile.common.model.log.EditedDocument; +import fr.dossierfacile.common.model.log.EditionType; import fr.dossierfacile.common.repository.LogRepository; import fr.dossierfacile.common.service.interfaces.LogService; import lombok.AllArgsConstructor; @@ -18,6 +20,7 @@ import org.springframework.stereotype.Service; import java.time.LocalDateTime; +import java.util.List; @Service @Slf4j @@ -43,12 +46,6 @@ public void saveLog(LogType logType, Long tenantId) { @Override public void saveLogWithTenantData(LogType logType, Tenant tenant) { - String content = null; - try { - content = objectMapper.writeValueAsString(deletedTenantCommonMapper.toDeletedTenantModel(tenant)); - } catch (Exception e) { - log.error("Cannot correclty record tenant information in tenant_log"); - } this.saveLog( Log.builder() .logType(logType) @@ -56,7 +53,7 @@ public void saveLogWithTenantData(LogType logType, Tenant tenant) { .creationDateTime(LocalDateTime.now()) .userApis(tenant.getTenantsUserApi().stream() .mapToLong(tenantUserApi -> tenantUserApi.getUserApi().getId()).toArray()) - .jsonProfile(content) + .jsonProfile(writeAsString(deletedTenantCommonMapper.toDeletedTenantModel(tenant))) .build() ); } @@ -67,20 +64,11 @@ public void saveDocumentEditedLog(Document document, Tenant editor, EditionType .logType(LogType.ACCOUNT_EDITED) .tenantId(editor.getId()) .creationDateTime(LocalDateTime.now()) - .logDetails(writeDocumentEditionDetails(document, editor.getId(), editionType)) + .logDetails(writeAsString(EditedDocument.from(document, editionType))) .build(); saveLog(log); } - private String writeDocumentEditionDetails(Document document, Long tenantId, EditionType editionType) { - try { - return objectMapper.writeValueAsString(EditedDocumentModel.from(document, editionType)); - } catch (JsonProcessingException e) { - log.error("Cannot write details of document edition from tenant {}", tenantId); - } - return null; - } - @Override public void savePartnerAccessRevocationLog(Tenant tenant, UserApi userApi) { Log log = Log.builder() @@ -92,4 +80,29 @@ public void savePartnerAccessRevocationLog(Tenant tenant, UserApi userApi) { saveLog(log); } + @Override + public void saveApplicationTypeChangedLog(List tenants, ApplicationType oldType, ApplicationType newType) { + if (oldType == newType) { + return; + } + for (Tenant tenant : tenants) { + Log log = Log.builder() + .logType(LogType.APPLICATION_TYPE_CHANGED) + .tenantId(tenant.getId()) + .creationDateTime(LocalDateTime.now()) + .logDetails(writeAsString(new ApplicationTypeChange(oldType, newType))) + .build(); + saveLog(log); + } + } + + private String writeAsString(Object object) { + try { + return objectMapper.writeValueAsString(object); + } catch (JsonProcessingException e) { + log.error("Cannot write log details as string"); + } + return null; + } + } diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java index 0da744611..0adec1576 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java @@ -3,8 +3,11 @@ import fr.dossierfacile.common.entity.Document; import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.entity.UserApi; +import fr.dossierfacile.common.enums.ApplicationType; import fr.dossierfacile.common.enums.LogType; -import fr.dossierfacile.common.model.EditionType; +import fr.dossierfacile.common.model.log.EditionType; + +import java.util.List; public interface LogService { @@ -16,4 +19,6 @@ public interface LogService { void savePartnerAccessRevocationLog(Tenant tenant, UserApi userApi); + void saveApplicationTypeChangedLog(List tenants, ApplicationType oldType, ApplicationType newType); + } diff --git a/dossierfacile-common-library/src/test/java/fr/dossierfacile/common/service/LogServiceImplTest.java b/dossierfacile-common-library/src/test/java/fr/dossierfacile/common/service/LogServiceImplTest.java index 729bced2b..c023c8fc5 100644 --- a/dossierfacile-common-library/src/test/java/fr/dossierfacile/common/service/LogServiceImplTest.java +++ b/dossierfacile-common-library/src/test/java/fr/dossierfacile/common/service/LogServiceImplTest.java @@ -9,12 +9,16 @@ import fr.dossierfacile.common.enums.DocumentCategory; import fr.dossierfacile.common.enums.DocumentSubCategory; import fr.dossierfacile.common.enums.LogType; -import fr.dossierfacile.common.model.EditionType; +import fr.dossierfacile.common.model.log.EditionType; import fr.dossierfacile.common.repository.LogRepository; import fr.dossierfacile.common.service.interfaces.LogService; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import java.util.List; + +import static fr.dossierfacile.common.enums.ApplicationType.ALONE; +import static fr.dossierfacile.common.enums.ApplicationType.GROUP; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -69,6 +73,17 @@ void should_save_revocation_log() { assertThat(savedLog.getUserApis()).containsExactly(2L); } + @Test + void should_application_type_change_log() { + logService.saveApplicationTypeChangedLog(List.of(tenantWithId(1L)), ALONE, GROUP); + + Log savedLog = getSavedLog(); + assertThat(savedLog.getLogType()).isEqualTo(LogType.APPLICATION_TYPE_CHANGED); + assertThat(savedLog.getTenantId()).isEqualTo(1L); + assertThat(savedLog.getLogDetails()).isEqualTo(""" + {"oldType":"ALONE","newType":"GROUP"}"""); + } + private static Tenant tenantWithId(long id) { return Tenant.builder().id(id).build(); } From 6abd2a0e8ea556476045db981d708e49dedb2ec8 Mon Sep 17 00:00:00 2001 From: Juliette de Rancourt Date: Wed, 20 Dec 2023 12:34:30 +0100 Subject: [PATCH 11/19] feat(tenant): Add step name in log when editing tenant names --- .../front/controller/RegisterController.java | 2 +- .../ApiPartnerRegisterController.java | 2 +- .../front/register/enums/StepRegister.java | 60 ++++++++++--------- .../common/model/log/EditedStep.java | 16 +++++ .../common/service/LogServiceImpl.java | 12 ++++ .../common/service/interfaces/LogService.java | 2 + 6 files changed, 65 insertions(+), 29 deletions(-) create mode 100644 dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditedStep.java diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/controller/RegisterController.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/controller/RegisterController.java index f072718e3..28bd38960 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/controller/RegisterController.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/controller/RegisterController.java @@ -65,7 +65,7 @@ public ResponseEntity confirmAccount(@PathVariable String token) { public ResponseEntity names(@Validated(Dossier.class) @RequestBody NamesForm namesForm) { var tenant = authenticationFacade.getTenant(namesForm.getTenantId() ); TenantModel tenantModel = tenantService.saveStepRegister(tenant, namesForm, StepRegister.NAMES); - logService.saveLog(LogType.ACCOUNT_EDITED, tenantModel.getId()); + logService.saveStepLog(tenantModel.getId(), StepRegister.NAMES.getClazz().getSimpleName()); Tenant loggedTenant = (namesForm.getTenantId() == null) ? tenant : authenticationFacade.getLoggedTenant(); return ok(tenantMapper.toTenantModel(loggedTenant)); } diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerRegisterController.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerRegisterController.java index 517a7d215..0275d335e 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerRegisterController.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerRegisterController.java @@ -54,7 +54,7 @@ public ResponseEntity account(@Validated(ApiPartner.class) @Request public ResponseEntity names(@Validated(ApiPartner.class) @RequestBody NamesForm namesForm) { var tenant = tenantService.findById(namesForm.getTenantId()); var tenantModel = tenantService.saveStepRegister(tenant, namesForm, StepRegister.NAMES); - logService.saveLog(LogType.ACCOUNT_EDITED, tenantModel.getId()); + logService.saveStepLog(tenantModel.getId(), StepRegister.NAMES.getClazz().getSimpleName()); return ok(tenantModel); } diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/enums/StepRegister.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/enums/StepRegister.java index bc50d5cfd..abd5f7797 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/enums/StepRegister.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/register/enums/StepRegister.java @@ -17,33 +17,39 @@ @Getter public enum StepRegister { - ACCOUNT_PARTNER_API(AccountApiPartner.class.getName()), - ACCOUNT(Account.class.getName()), - NAMES(Names.class.getName()), - APPLICATION_V1(ApplicationV1.class.getName()), - APPLICATION(Application.class.getName()), - HONOR_DECLARATION(HonorDeclaration.class.getName()), - DOCUMENT_IDENTIFICATION(DocumentIdentification.class.getName()), - DOCUMENT_RESIDENCY(DocumentResidency.class.getName()), - DOCUMENT_PROFESSIONAL(DocumentProfessional.class.getName()), - DOCUMENT_FINANCIAL(DocumentFinancial.class.getName()), - DOCUMENT_TAX(DocumentTax.class.getName()), - NAME_GUARANTOR_NATURAL_PERSON(NameGuarantorNaturalPerson.class.getName()), - DOCUMENT_IDENTIFICATION_GUARANTOR_NATURAL_PERSON_FILE(DocumentIdentificationGuarantorNaturalPersonFile.class.getName()), - DOCUMENT_IDENTIFICATION_GUARANTOR_NATURAL_PERSON(DocumentIdentificationGuarantorNaturalPerson.class.getName()), - DOCUMENT_RESIDENCY_GUARANTOR_NATURAL_PERSON(DocumentResidencyGuarantorNaturalPerson.class.getName()), - DOCUMENT_PROFESSIONAL_GUARANTOR_NATURAL_PERSON(DocumentProfessionalGuarantorNaturalPerson.class.getName()), - DOCUMENT_FINANCIAL_GUARANTOR_NATURAL_PERSON(DocumentFinancialGuarantorNaturalPerson.class.getName()), - DOCUMENT_TAX_GUARANTOR_NATURAL_PERSON(DocumentTaxGuarantorNaturalPerson.class.getName()), - DOCUMENT_IDENTIFICATION_GUARANTOR_ORGANISM(DocumentIdentificationGuarantorOrganism.class.getName()), - DOCUMENT_IDENTIFICATION_GUARANTOR_LEGAL_PERSON(DocumentIdentificationGuarantorLegalPerson.class.getName()), - DOCUMENT_IDENTIFICATION_REPRESENTANT_GUARANTOR_LEGAL_PERSON(DocumentIdentificationRepresentanGuarantorLegalPerson.class.getName()), - GUARANTOR_TYPE(GuarantorType.class.getName()), - NAME_GUARANTOR_LEGAL_PERSON(NameGuarantorLegalPerson.class.getName()), - NAME_IDENTIFICATION_REPRESENTANT_GUARANTOR_LEGAL_PERSON(NameRepresentantGuarantorLegalPerson.class.getName()); - private final String label; + ACCOUNT_PARTNER_API(AccountApiPartner.class), + ACCOUNT(Account.class), + NAMES(Names.class), + APPLICATION_V1(ApplicationV1.class), + APPLICATION(Application.class), + HONOR_DECLARATION(HonorDeclaration.class), + DOCUMENT_IDENTIFICATION(DocumentIdentification.class), + DOCUMENT_RESIDENCY(DocumentResidency.class), + DOCUMENT_PROFESSIONAL(DocumentProfessional.class), + DOCUMENT_FINANCIAL(DocumentFinancial.class), + DOCUMENT_TAX(DocumentTax.class), + NAME_GUARANTOR_NATURAL_PERSON(NameGuarantorNaturalPerson.class), + DOCUMENT_IDENTIFICATION_GUARANTOR_NATURAL_PERSON_FILE(DocumentIdentificationGuarantorNaturalPersonFile.class), + DOCUMENT_IDENTIFICATION_GUARANTOR_NATURAL_PERSON(DocumentIdentificationGuarantorNaturalPerson.class), + DOCUMENT_RESIDENCY_GUARANTOR_NATURAL_PERSON(DocumentResidencyGuarantorNaturalPerson.class), + DOCUMENT_PROFESSIONAL_GUARANTOR_NATURAL_PERSON(DocumentProfessionalGuarantorNaturalPerson.class), + DOCUMENT_FINANCIAL_GUARANTOR_NATURAL_PERSON(DocumentFinancialGuarantorNaturalPerson.class), + DOCUMENT_TAX_GUARANTOR_NATURAL_PERSON(DocumentTaxGuarantorNaturalPerson.class), + DOCUMENT_IDENTIFICATION_GUARANTOR_ORGANISM(DocumentIdentificationGuarantorOrganism.class), + DOCUMENT_IDENTIFICATION_GUARANTOR_LEGAL_PERSON(DocumentIdentificationGuarantorLegalPerson.class), + DOCUMENT_IDENTIFICATION_REPRESENTANT_GUARANTOR_LEGAL_PERSON(DocumentIdentificationRepresentanGuarantorLegalPerson.class), + GUARANTOR_TYPE(GuarantorType.class), + NAME_GUARANTOR_LEGAL_PERSON(NameGuarantorLegalPerson.class), + NAME_IDENTIFICATION_REPRESENTANT_GUARANTOR_LEGAL_PERSON(NameRepresentantGuarantorLegalPerson.class); - StepRegister(String label) { - this.label = label; + private final Class clazz; + + StepRegister(Class clazz) { + this.clazz = clazz; + } + + public String getLabel() { + return clazz.getName(); } + } diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditedStep.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditedStep.java new file mode 100644 index 000000000..70620da8c --- /dev/null +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/log/EditedStep.java @@ -0,0 +1,16 @@ +package fr.dossierfacile.common.model.log; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EditedStep { + + private String step; + +} diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java index 0a49de0ef..0ccd227bc 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/LogServiceImpl.java @@ -12,6 +12,7 @@ import fr.dossierfacile.common.mapper.DeletedTenantCommonMapper; import fr.dossierfacile.common.model.log.ApplicationTypeChange; import fr.dossierfacile.common.model.log.EditedDocument; +import fr.dossierfacile.common.model.log.EditedStep; import fr.dossierfacile.common.model.log.EditionType; import fr.dossierfacile.common.repository.LogRepository; import fr.dossierfacile.common.service.interfaces.LogService; @@ -44,6 +45,17 @@ public void saveLog(LogType logType, Long tenantId) { this.saveLog(new Log(logType, tenantId)); } + @Override + public void saveStepLog(Long tenantId, String stepName) { + Log log = Log.builder() + .logType(LogType.ACCOUNT_EDITED) + .tenantId(tenantId) + .creationDateTime(LocalDateTime.now()) + .logDetails(writeAsString(new EditedStep(stepName))) + .build(); + saveLog(log); + } + @Override public void saveLogWithTenantData(LogType logType, Tenant tenant) { this.saveLog( diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java index 0adec1576..249c3bbed 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/service/interfaces/LogService.java @@ -13,6 +13,8 @@ public interface LogService { void saveLog(LogType logType, Long tenantId); + void saveStepLog(Long tenantId, String step); + void saveLogWithTenantData(LogType logType, Tenant tenant); void saveDocumentEditedLog(Document document, Tenant editor, EditionType editionType); From 469f5e79fd47c424029098fcd639d31adb949589 Mon Sep 17 00:00:00 2001 From: Juliette de Rancourt Date: Wed, 20 Dec 2023 17:41:10 +0100 Subject: [PATCH 12/19] feat(tenant): Update partner synchronization endpoints to return revoked accesses if asked (#675) --- dossierfacile-api-tenant/pom.xml | 4 ++ .../dfc/controller/DfcTenantsController.java | 17 +++++-- .../ApiPartnerTenantController.java | 24 +++++++--- .../service/PartnerAccessServiceImpl.java | 7 ++- .../api/front/service/TenantServiceImpl.java | 5 +- .../service/interfaces/TenantService.java | 2 +- .../controller/DfcTenantsControllerTest.java | 41 ++++++++++++++++ .../ApiPartnerTenantControllerTest.java | 44 +++++++++++++++++ .../common/model/TenantUpdate.java | 1 + .../repository/TenantCommonRepository.java | 47 +++++++++++-------- 10 files changed, 157 insertions(+), 35 deletions(-) create mode 100644 dossierfacile-api-tenant/src/test/java/fr/dossierfacile/api/front/dfc/controller/DfcTenantsControllerTest.java create mode 100644 dossierfacile-api-tenant/src/test/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerTenantControllerTest.java diff --git a/dossierfacile-api-tenant/pom.xml b/dossierfacile-api-tenant/pom.xml index cc1eaa982..951b75bdf 100644 --- a/dossierfacile-api-tenant/pom.xml +++ b/dossierfacile-api-tenant/pom.xml @@ -233,6 +233,10 @@ logstash-logback-encoder 7.3 + + org.springframework.boot + spring-boot-test + diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/dfc/controller/DfcTenantsController.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/dfc/controller/DfcTenantsController.java index 2451b645b..fe284e979 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/dfc/controller/DfcTenantsController.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/dfc/controller/DfcTenantsController.java @@ -23,6 +23,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.util.UriComponentsBuilder; import java.time.LocalDateTime; import java.util.List; @@ -46,13 +47,21 @@ public class DfcTenantsController { @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity, ListMetadata>> list(@RequestParam(value = "after", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) LocalDateTime after, @RequestParam(value = "limit", defaultValue = "1000") Long limit, - @RequestParam(value = "includeDeleted", defaultValue = "false") boolean includeDeleted + @RequestParam(value = "includeDeleted", defaultValue = "false") boolean includeDeleted, + @RequestParam(value = "includeRevoked", defaultValue = "false") boolean includeRevoked ) { UserApi userApi = clientAuthenticationFacade.getClient(); - List result = tenantService.findTenantUpdateByLastUpdateAndPartner(after, userApi, limit, includeDeleted); - LocalDateTime nextTimeToken = (result.size() == 0) ? after : result.get(result.size() - 1).getLastUpdateDate(); + List result = tenantService.findTenantUpdateByLastUpdateAndPartner(after, userApi, limit, includeDeleted, includeRevoked); + + LocalDateTime nextTimeToken = result.isEmpty() ? after : result.get(result.size() - 1).getLastUpdateDate(); + + String nextLink = UriComponentsBuilder.fromPath(PATH) + .queryParam("limit", limit) + .queryParam("after", nextTimeToken) + .queryParam("includeDeleted", includeDeleted) + .queryParam("includeRevoked", includeRevoked) + .build().encode().toUriString(); - String nextLink = PATH + "?limit=" + limit + "&after=" + nextTimeToken + "&includeDeleted=" + includeDeleted; return ok(ResponseWrapper., ListMetadata>builder() .metadata(ListMetadata.builder() .limit(limit) diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerTenantController.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerTenantController.java index 6aa885a1c..f376acd92 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerTenantController.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerTenantController.java @@ -24,6 +24,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.util.UriComponentsBuilder; import java.time.LocalDateTime; import java.util.List; @@ -46,27 +47,36 @@ public class ApiPartnerTenantController { public ResponseEntity, ListMetadata>> list(@RequestParam(value = "after", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) LocalDateTime after, @RequestParam(value = "limit", defaultValue = "1000") Long limit, @RequestParam(value = "orderBy", defaultValue = "LAST_UPDATE_DATE") TenantSortType orderBy, - @RequestParam(value = "includeDeleted", defaultValue = "false") boolean includeDeleted + @RequestParam(value = "includeDeleted", defaultValue = "false") boolean includeDeleted, + @RequestParam(value = "includeRevoked", defaultValue = "false") boolean includeRevoked ) { + System.out.println("after = " + after); UserApi userApi = clientAuthenticationFacade.getClient(); List result; LocalDateTime nextTimeToken; switch (orderBy) { case CREATION_DATE -> { - if (includeDeleted){ - throw new IllegalArgumentException("includeDelete is not available with creationDate order"); + if (includeDeleted || includeRevoked) { + throw new IllegalArgumentException("includeDelete and includeRevoked are not available with creationDate order"); } result = tenantService.findTenantUpdateByCreatedAndPartner(after, userApi, limit); - nextTimeToken = (result.size() == 0) ? after : result.get(result.size() - 1).getCreationDate(); + nextTimeToken = result.isEmpty() ? after : result.get(result.size() - 1).getCreationDate(); } case LAST_UPDATE_DATE -> { - result = tenantService.findTenantUpdateByLastUpdateAndPartner(after, userApi, limit, includeDeleted); - nextTimeToken = (result.size() == 0) ? after : result.get(result.size() - 1).getLastUpdateDate(); + result = tenantService.findTenantUpdateByLastUpdateAndPartner(after, userApi, limit, includeDeleted, includeRevoked); + nextTimeToken = result.isEmpty() ? after : result.get(result.size() - 1).getLastUpdateDate(); } default -> throw new IllegalArgumentException(); } - String nextLink = "/api-partner/tenant?limit=" + limit + "&orderBy=" + orderBy + "&after=" + nextTimeToken + "&includeDeleted=" + includeDeleted; + String nextLink = UriComponentsBuilder.fromPath("/api-partner/tenant") + .queryParam("limit", limit) + .queryParam("orderBy", orderBy) + .queryParam("after", nextTimeToken) + .queryParam("includeDeleted", includeDeleted) + .queryParam("includeRevoked", includeRevoked) + .build().encode().toUriString(); + return ok(ResponseWrapper., ListMetadata>builder() .metadata(ListMetadata.builder() .limit(limit) diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/PartnerAccessServiceImpl.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/PartnerAccessServiceImpl.java index abaedcc9e..cd6673b36 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/PartnerAccessServiceImpl.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/PartnerAccessServiceImpl.java @@ -8,6 +8,7 @@ import fr.dossierfacile.common.entity.Tenant; import fr.dossierfacile.common.entity.TenantUserApi; import fr.dossierfacile.common.entity.UserApi; +import fr.dossierfacile.common.repository.TenantCommonRepository; import fr.dossierfacile.common.repository.TenantUserApiRepository; import fr.dossierfacile.common.service.interfaces.LogService; import fr.dossierfacile.common.service.interfaces.PartnerCallBackService; @@ -15,6 +16,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import java.time.LocalDateTime; import java.util.List; @Slf4j @@ -26,6 +28,7 @@ public class PartnerAccessServiceImpl implements PartnerAccessService { private final TenantUserApiRepository tenantUserApiRepository; private final PartnerCallBackService partnerCallBackService; + private final TenantCommonRepository tenantRepository; private final PartnerAccessMapper partnerAccessMapper; private final KeycloakService keycloakService; private final MailService mailService; @@ -46,8 +49,10 @@ public void deleteAccess(Tenant tenant, Long userApiId) { } private void deleteAccess(TenantUserApi tenantUserApi) { - Tenant tenant = tenantUserApi.getTenant(); UserApi userApi = tenantUserApi.getUserApi(); + Tenant tenant = tenantUserApi.getTenant(); + tenant.setLastUpdateDate(LocalDateTime.now()); + tenantRepository.save(tenant); tenantUserApiRepository.delete(tenantUserApi); partnerCallBackService.sendRevokedAccessCallback(tenant, userApi); diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/TenantServiceImpl.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/TenantServiceImpl.java index dcf9bb69b..b904759b6 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/TenantServiceImpl.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/TenantServiceImpl.java @@ -156,9 +156,8 @@ public List findTenantUpdateByCreatedAndPartner(LocalDateTime from } @Override - public List findTenantUpdateByLastUpdateAndPartner(LocalDateTime since, UserApi userApi, Long limit, boolean includeDeleted) { - return includeDeleted? tenantRepository.findTenantUpdateWithDeletedByLastUpdateAndPartner(since, userApi.getId(), limit) : - tenantRepository.findTenantUpdateByLastUpdateAndPartner(since, userApi.getId(), limit); + public List findTenantUpdateByLastUpdateAndPartner(LocalDateTime since, UserApi userApi, Long limit, boolean includeDeleted, boolean includeRevoked) { + return tenantRepository.findTenantUpdateByLastUpdateAndPartner(since, userApi.getId(), limit, includeDeleted, includeRevoked); } @Override diff --git a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/interfaces/TenantService.java b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/interfaces/TenantService.java index b4c47e5f8..829fd70a1 100644 --- a/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/interfaces/TenantService.java +++ b/dossierfacile-api-tenant/src/main/java/fr/dossierfacile/api/front/service/interfaces/TenantService.java @@ -33,7 +33,7 @@ public interface TenantService { List findTenantUpdateByCreatedAndPartner(LocalDateTime from, UserApi userApi, Long limit); - List findTenantUpdateByLastUpdateAndPartner(LocalDateTime from, UserApi userApi, Long limit, boolean includeDeleted); + List findTenantUpdateByLastUpdateAndPartner(LocalDateTime from, UserApi userApi, Long limit, boolean includeDeleted, boolean includeRevoked); void sendFileByMail(Tenant tenant, String email, String shareType); } diff --git a/dossierfacile-api-tenant/src/test/java/fr/dossierfacile/api/front/dfc/controller/DfcTenantsControllerTest.java b/dossierfacile-api-tenant/src/test/java/fr/dossierfacile/api/front/dfc/controller/DfcTenantsControllerTest.java new file mode 100644 index 000000000..48303f619 --- /dev/null +++ b/dossierfacile-api-tenant/src/test/java/fr/dossierfacile/api/front/dfc/controller/DfcTenantsControllerTest.java @@ -0,0 +1,41 @@ +package fr.dossierfacile.api.front.dfc.controller; + +import fr.dossierfacile.api.front.security.interfaces.ClientAuthenticationFacade; +import fr.dossierfacile.api.front.service.interfaces.TenantService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +class DfcTenantsControllerTest { + + private MockMvc mvc; + + @BeforeEach + public void setUp() { + var controller = new DfcTenantsController(mock(ClientAuthenticationFacade.class), mock(TenantService.class), null, null); + mvc = MockMvcBuilders.standaloneSetup(controller).build(); + } + + @Test + void should_add_response_metadata() throws Exception { + var request = get("/dfc/api/v1/tenants") + .queryParam("after", "2020-01-31T10:30:00.000-05:00") + .queryParam("limit", "10") + .queryParam("includeDeleted", "true"); + + String contentAsString = mvc.perform(request) + .andExpect(status().isOk()) + .andReturn() + .getResponse().getContentAsString(); + + assertThat(contentAsString).isEqualToIgnoringNewLines(""" + {"data":[],"metadata":{"limit":10,"resultCount":0,"nextLink":"/dfc/api/v1/tenants?limit=10&after=2020-01-31T10:30&includeDeleted=true&includeRevoked=false"}}"""); + } + +} \ No newline at end of file diff --git a/dossierfacile-api-tenant/src/test/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerTenantControllerTest.java b/dossierfacile-api-tenant/src/test/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerTenantControllerTest.java new file mode 100644 index 000000000..612db3a5f --- /dev/null +++ b/dossierfacile-api-tenant/src/test/java/fr/dossierfacile/api/front/partner/controller/ApiPartnerTenantControllerTest.java @@ -0,0 +1,44 @@ +package fr.dossierfacile.api.front.partner.controller; + +import fr.dossierfacile.api.front.security.interfaces.ClientAuthenticationFacade; +import fr.dossierfacile.api.front.service.interfaces.TenantService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import java.time.LocalDateTime; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +class ApiPartnerTenantControllerTest { + + private MockMvc mvc; + + @BeforeEach + public void setUp() { + var controller = new ApiPartnerTenantController(mock(ClientAuthenticationFacade.class), mock(TenantService.class), null, null); + mvc = MockMvcBuilders.standaloneSetup(controller).build(); + } + + @Test + void should_add_response_metadata() throws Exception { + var request = get("/api-partner/tenant") + .queryParam("after", "2020-01-31T10:30:00.000-05:00") + .queryParam("limit", "10") + .queryParam("includeDeleted", "true"); + + String contentAsString = mvc.perform(request) + .andExpect(status().isOk()) + .andReturn() + .getResponse().getContentAsString(); + + assertThat(contentAsString).isEqualToIgnoringNewLines(""" + {"data":[],"metadata":{"limit":10,"resultCount":0,"nextLink":"/api-partner/tenant?limit=10&orderBy=LAST_UPDATE_DATE&after=2020-01-31T10:30&includeDeleted=true&includeRevoked=false"}}"""); + } + +} \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/TenantUpdate.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/TenantUpdate.java index 7691e7e3b..11bab46ba 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/TenantUpdate.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/model/TenantUpdate.java @@ -8,4 +8,5 @@ public interface TenantUpdate { LocalDateTime getLastUpdateDate(); LocalDateTime getCreationDate(); LocalDateTime getDeletionDate(); + LocalDateTime getRevocationDate(); } \ No newline at end of file diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/TenantCommonRepository.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/TenantCommonRepository.java index 8a40d6035..37da0b878 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/TenantCommonRepository.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/TenantCommonRepository.java @@ -215,20 +215,39 @@ SELECT COUNT(t.id) @Query(value = """ SELECT * FROM ( SELECT tenant_id as id, - null as apartmentSharingId, - null as lastUpdateDate, - null as creationDate, - creation_date as deletionDate + CAST(null AS bigint) as apartmentSharingId, + CAST(null AS timestamp) as lastUpdateDate, + CAST(null AS timestamp) as creationDate, + creation_date as deletionDate, + CAST(null AS timestamp) as revocationDate FROM tenant_log WHERE (CAST(CAST(:lastUpdateFrom AS text) AS timestamp) IS NULL OR creation_date > CAST(CAST(:lastUpdateFrom AS text) AS timestamp)) + AND log_type = 'ACCOUNT_DELETE' AND :partnerId = ANY (user_apis) + AND :includeDeleted + UNION + (SELECT tenant_id as id, + CAST(null AS bigint) as apartmentSharingId, + CAST(null AS timestamp) as lastUpdateDate, + CAST(null AS timestamp) as creationDate, + CAST(null AS timestamp) as deletionDate, + creation_date as revocationDate + FROM tenant_log tl + WHERE (CAST(CAST(:lastUpdateFrom AS text) AS timestamp) IS NULL + OR creation_date > CAST(CAST(:lastUpdateFrom AS text) AS timestamp)) + AND log_type = 'PARTNER_ACCESS_REVOKED' + AND :partnerId = ANY (user_apis) + AND NOT EXISTS (SELECT 1 FROM tenant_userapi tua WHERE tua.tenant_id = tl.tenant_id AND tua.userapi_id = :partnerId) + AND :includeRevoked + ORDER BY revocationDate DESC LIMIT 1) UNION SELECT t.id as id, t.apartment_sharing_id as apartmentSharingId, t.last_update_date as lastUpdateDate, ua.creation_date as creationDate, - null as deletionDate + CAST(null AS timestamp) as deletionDate, + CAST(null AS timestamp) as revocationDate FROM tenant t INNER JOIN user_account ua ON ua.id = t.id INNER JOIN tenant_userapi tua ON tua.tenant_id = t.id @@ -238,20 +257,10 @@ OR creation_date > CAST(CAST(:lastUpdateFrom AS text) AS timestamp)) ORDER BY lastUpdateDate ASC LIMIT :limit """, nativeQuery = true) - List findTenantUpdateWithDeletedByLastUpdateAndPartner(@Param("lastUpdateFrom") LocalDateTime from, @Param("partnerId") Long id, @Param("limit") Long limit); - - @Query(value = """ - SELECT t.id as id, t.apartment_sharing_id as apartmentSharingId, t.last_update_date as lastUpdateDate, ua.creation_date as creationDate - FROM tenant t - INNER JOIN user_account ua ON ua.id = t.id - INNER JOIN tenant_userapi tua ON tua.tenant_id = t.id - WHERE tua.userapi_id = :partnerId - AND ( CAST( CAST(:lastUpdateFrom AS text) AS timestamp) IS NULL OR t.last_update_date > CAST( CAST(:lastUpdateFrom AS text) AS timestamp)) - ORDER BY t.last_update_date ASC - LIMIT :limit - """, nativeQuery = true - ) - List findTenantUpdateByLastUpdateAndPartner(@Param("lastUpdateFrom") LocalDateTime from, @Param("partnerId") Long id, @Param("limit") Long limit); + List findTenantUpdateByLastUpdateAndPartner(@Param("lastUpdateFrom") LocalDateTime from, + @Param("partnerId") Long id, @Param("limit") Long limit, + @Param("includeDeleted") boolean includeDeleted, + @Param("includeRevoked") boolean includeRevoked); @Query(value = """ SELECT t.id as id, t.apartment_sharing_id as apartmentSharingId, t.last_update_date as lastUpdateDate, ua.creation_date as creationDate From 7f73c659c19a494586b24b72ec3db99d91613134 Mon Sep 17 00:00:00 2001 From: Juliette de Rancourt Date: Thu, 21 Dec 2023 11:13:34 +0100 Subject: [PATCH 13/19] chore: Upgrade bouncycastle versions --- dossierfacile-api-tenant/pom.xml | 4 ++-- dossierfacile-bo/pom.xml | 4 ++-- dossierfacile-process-file/pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dossierfacile-api-tenant/pom.xml b/dossierfacile-api-tenant/pom.xml index 951b75bdf..5f6025dc9 100644 --- a/dossierfacile-api-tenant/pom.xml +++ b/dossierfacile-api-tenant/pom.xml @@ -166,8 +166,8 @@ org.bouncycastle - bcprov-jdk15on - 1.68 + bcprov-jdk18on + 1.77 diff --git a/dossierfacile-bo/pom.xml b/dossierfacile-bo/pom.xml index 69665f3b0..b45e0da58 100644 --- a/dossierfacile-bo/pom.xml +++ b/dossierfacile-bo/pom.xml @@ -204,8 +204,8 @@ org.bouncycastle - bcprov-jdk15on - 1.68 + bcprov-jdk18on + 1.77 diff --git a/dossierfacile-process-file/pom.xml b/dossierfacile-process-file/pom.xml index cfb5cd9ba..f62aba52c 100644 --- a/dossierfacile-process-file/pom.xml +++ b/dossierfacile-process-file/pom.xml @@ -124,7 +124,7 @@ org.bouncycastle bcprov-jdk18on - 1.74 + 1.77 org.apache.commons From d80a6d90cda1b47455289d6683cd208322159a34 Mon Sep 17 00:00:00 2001 From: Fabien Date: Thu, 21 Dec 2023 22:57:41 +0100 Subject: [PATCH 14/19] feat: clean failed pdf - synchronize partner and send mail --- .../bo/repository/DocumentRepository.java | 5 +- .../fr/gouv/bo/service/DocumentService.java | 4 +- .../main/resources/templates/bo/index.html | 2 +- .../scheduler/tasks/TaskName.java | 3 +- .../document/DocumentDeleteMailService.java | 56 ++++++++++++ .../DocumentModel.java | 2 +- .../DocumentRepository.java | 19 +++- .../tasks/document/DocumentTask.java | 88 +++++++++++++++++++ .../document/PartnerCallbackService.java | 33 +++++++ .../DocumentToProcessTask.java | 56 ------------ .../BackupFilesTask.java | 2 +- .../src/main/resources/application.properties | 6 +- 12 files changed, 206 insertions(+), 70 deletions(-) create mode 100644 dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentDeleteMailService.java rename dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/{documenttoprocess => document}/DocumentModel.java (64%) rename dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/{documenttoprocess => document}/DocumentRepository.java (54%) create mode 100644 dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentTask.java create mode 100644 dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/PartnerCallbackService.java delete mode 100644 dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentToProcessTask.java diff --git a/dossierfacile-bo/src/main/java/fr/gouv/bo/repository/DocumentRepository.java b/dossierfacile-bo/src/main/java/fr/gouv/bo/repository/DocumentRepository.java index 0c2e8d0df..0169dcba3 100644 --- a/dossierfacile-bo/src/main/java/fr/gouv/bo/repository/DocumentRepository.java +++ b/dossierfacile-bo/src/main/java/fr/gouv/bo/repository/DocumentRepository.java @@ -21,11 +21,10 @@ public interface DocumentRepository extends JpaRepository { @Query(value = """ SELECT d.id FROM Document d - WHERE d.document_status = 'TO_PROCESS' - AND (d.last_modified_date IS NULL OR d.last_modified_date < :to) + WHERE (d.last_modified_date IS NULL OR d.last_modified_date < :to) AND d.watermark_file_id IS NULL """, nativeQuery = true) - List findToProcessWithoutPDFToDate(@Param("to") LocalDateTime toDateTime); + List findWithoutPDFToDate(@Param("to") LocalDateTime toDateTime); @Modifying @Query("UPDATE Document d SET d.documentDeniedReasons = :documentDeniedReasons where d.id = :documentId") diff --git a/dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java b/dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java index 3fa4edac7..71cc84728 100644 --- a/dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java +++ b/dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java @@ -94,8 +94,8 @@ public void initializeFieldsToProcessPdfGeneration(Document document) { public void regenerateFailedPdfDocumentsUsingButtonRequest() { synchronized (this) { LocalDateTime oneHourAgo = LocalDateTime.now().minusHours(1); - List documents = documentRepository.findToProcessWithoutPDFToDate(oneHourAgo); - log.info("Regenerate [{}] Failed PDF in TO PROCESS status", documents.size()); + List documents = documentRepository.findWithoutPDFToDate(oneHourAgo); + log.info("Regenerate [{}] Failed PDF in all status", documents.size()); documents.forEach(documentId -> { try { diff --git a/dossierfacile-bo/src/main/resources/templates/bo/index.html b/dossierfacile-bo/src/main/resources/templates/bo/index.html index 5b1e04698..1ec559aa8 100644 --- a/dossierfacile-bo/src/main/resources/templates/bo/index.html +++ b/dossierfacile-bo/src/main/resources/templates/bo/index.html @@ -80,7 +80,7 @@
diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/TaskName.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/TaskName.java index 1e35796be..d89694e75 100644 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/TaskName.java +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/TaskName.java @@ -8,6 +8,7 @@ public enum TaskName { TENANT_DELETION, STORAGE_FILES_BACKUP, STORAGE_FILES_DELETION, - PDF_GENERATION + PDF_GENERATION, + DELETE_FAILED_DOCUMENT; } diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentDeleteMailService.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentDeleteMailService.java new file mode 100644 index 000000000..024523761 --- /dev/null +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentDeleteMailService.java @@ -0,0 +1,56 @@ +package fr.dossierfacile.scheduler.tasks.document; + +import fr.dossierfacile.common.entity.Document; +import fr.dossierfacile.common.entity.Tenant; +import fr.dossierfacile.common.repository.TenantCommonRepository; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import sendinblue.ApiException; +import sibApi.TransactionalEmailsApi; +import sibModel.SendSmtpEmail; +import sibModel.SendSmtpEmailTo; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +@Service +@RequiredArgsConstructor +@Slf4j +public class DocumentDeleteMailService { + private final TransactionalEmailsApi apiInstance; + private final TenantCommonRepository tenantCommonRepository; + @Value("${sendinblue.template.id.deleted.document.with.failed.pdf}") + private Long templateDeletedDocumentWithFailedPdf; + + @Transactional(readOnly = true) + public void sendMailWithDocumentFailed(Long tenantId, List documents) { + log.debug("Send a email to {} with {} documents", tenantId, documents.size()); + Tenant tenant = tenantCommonRepository.findById(tenantId).get(); + if (isNotBlank(tenant.getEmail())) { + Map variables = new HashMap<>(); + variables.put("PRENOM", tenant.getFirstName()); + + SendSmtpEmailTo sendSmtpEmailTo = new SendSmtpEmailTo(); + sendSmtpEmailTo.setEmail(tenant.getEmail()); + + SendSmtpEmail sendSmtpEmail = new SendSmtpEmail(); + sendSmtpEmail.templateId(templateDeletedDocumentWithFailedPdf); + sendSmtpEmail.params(variables); + sendSmtpEmail.to(Collections.singletonList(sendSmtpEmailTo)); + + try { + apiInstance.sendTransacEmail(sendSmtpEmail); + } catch (ApiException e) { + log.error("Email api exception", e); + } + } + + } +} diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentModel.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentModel.java similarity index 64% rename from dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentModel.java rename to dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentModel.java index 2d0567776..2981a43fa 100644 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentModel.java +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentModel.java @@ -1,4 +1,4 @@ -package fr.dossierfacile.scheduler.tasks.documenttoprocess; +package fr.dossierfacile.scheduler.tasks.document; import lombok.Builder; diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentRepository.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentRepository.java similarity index 54% rename from dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentRepository.java rename to dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentRepository.java index 76e24ca63..09af58ee6 100644 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentRepository.java +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentRepository.java @@ -1,4 +1,4 @@ -package fr.dossierfacile.scheduler.tasks.documenttoprocess; +package fr.dossierfacile.scheduler.tasks.document; import fr.dossierfacile.common.entity.Document; import org.springframework.data.jpa.repository.JpaRepository; @@ -14,13 +14,24 @@ public interface DocumentRepository extends JpaRepository { @Query(value = """ SELECT * FROM document - WHERE document_status = 'TO_PROCESS' - AND watermark_file_id IS NULL + WHERE watermark_file_id IS NULL AND (last_modified_date IS NULL OR last_modified_date < :to) ORDER BY CASE WHEN last_modified_date IS NULL THEN 1 ELSE 0 END, last_modified_date DESC LIMIT 200; """, nativeQuery = true) - List findToProcessWithoutPDFToDate(@Param("to") LocalDateTime toDateTime); + List findWithoutPDFToDate(@Param("to") LocalDateTime toDateTime); + + @Query(""" + SELECT d + FROM Document d + LEFT JOIN FETCH d.guarantor g + WHERE d.watermarkFile IS NULL + AND ((d.lastModifiedDate IS NOT NULL AND d.lastModifiedDate < :to) + OR (d.lastModifiedDate IS NULL AND d.creationDateTime < :to)) + ORDER BY d.lastModifiedDate DESC + """) + List findDocumentWithoutPDFToDate(@Param("to") LocalDateTime toDateTime); + } diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentTask.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentTask.java new file mode 100644 index 000000000..991dde74e --- /dev/null +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/DocumentTask.java @@ -0,0 +1,88 @@ +package fr.dossierfacile.scheduler.tasks.document; + +import com.google.gson.Gson; +import fr.dossierfacile.common.entity.Document; +import fr.dossierfacile.common.entity.Tenant; +import fr.dossierfacile.scheduler.LoggingContext; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.amqp.core.AmqpTemplate; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageProperties; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.time.LocalDateTime; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static fr.dossierfacile.scheduler.tasks.TaskName.DELETE_FAILED_DOCUMENT; +import static fr.dossierfacile.scheduler.tasks.TaskName.PDF_GENERATION; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DocumentTask { + + private final AmqpTemplate amqpTemplate; + private final Gson gson; + private final DocumentRepository documentRepository; + private final PartnerCallbackService partnerCallbackService; + private final DocumentDeleteMailService documentDeleteMailService; + + @Value("${rabbitmq.exchange.pdf.generator}") + private String exchangePdfGenerator; + @Value("${rabbitmq.routing.key.pdf.generator.watermark-document}") + private String routingKeyPdfGenerator; + @Value("${document.pdf.failed.delay.before.delete.hours}") + private Long delayBeforeDeleteHours; + + @Scheduled(cron = "${cron.process.pdf.generation.failed}") + public void reLaunchFailedPDFGeneration() { + LoggingContext.startTask(PDF_GENERATION); + LocalDateTime now = LocalDateTime.now(); + LocalDateTime toDateTime = now.minusMinutes(30); + documentRepository.findWithoutPDFToDate(toDateTime).forEach(this::sendForPDFGeneration); + LoggingContext.endTask(); + } + + @Scheduled(cron = "${cron.delete.document.with.failed.pdf}") + public void deleteDocumentWithFailedPdfGeneration() { + LoggingContext.startTask(DELETE_FAILED_DOCUMENT); + LocalDateTime now = LocalDateTime.now(); + LocalDateTime toDateTime = now.minusHours(delayBeforeDeleteHours); + + List documents = documentRepository.findDocumentWithoutPDFToDate(toDateTime); + if (CollectionUtils.isEmpty(documents)) { + log.info("There is not file with empty pdf"); + } else { + Map> tenantDocuments = documents.stream() + .collect(Collectors.groupingBy(d -> + Optional.ofNullable(d.getTenant()) + .orElseGet(() -> d.getGuarantor().getTenant()) + )); + tenantDocuments.forEach((tenant, docs) -> documentDeleteMailService.sendMailWithDocumentFailed(tenant.getId(), docs)); + documentRepository.deleteAll(documents); + tenantDocuments.forEach((tenant, docs) -> partnerCallbackService.sendPartnerCallback(tenant.getId())); + } + LoggingContext.endTask(); + } + + private void sendForPDFGeneration(Document document) { + log.debug("Sending document with ID [{}] for pdf generation", document.getId()); + amqpTemplate.send(exchangePdfGenerator, routingKeyPdfGenerator, messageWithId(document.getId())); + } + + private Message messageWithId(Long id) { + MessageProperties properties = new MessageProperties(); + properties.setHeader("timestamp", System.currentTimeMillis()); + return new Message( + gson.toJson(Collections.singletonMap("id", String.valueOf(id))).getBytes(), + properties); + } +} \ No newline at end of file diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/PartnerCallbackService.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/PartnerCallbackService.java new file mode 100644 index 000000000..52669a5ff --- /dev/null +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/document/PartnerCallbackService.java @@ -0,0 +1,33 @@ +package fr.dossierfacile.scheduler.tasks.document; + +import fr.dossierfacile.common.entity.Tenant; +import fr.dossierfacile.common.enums.PartnerCallBackType; +import fr.dossierfacile.common.enums.TenantFileStatus; +import fr.dossierfacile.common.repository.TenantCommonRepository; +import fr.dossierfacile.common.service.interfaces.PartnerCallBackService; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.util.Optional; + +@Slf4j +@Service +@AllArgsConstructor +public class PartnerCallbackService { + private final TenantCommonRepository tenantCommonRepository; + private final PartnerCallBackService partnerCallBackService; + + @Transactional + public void sendPartnerCallback(Long tenantId) { + log.debug("Send Callback to partners"); + Optional tenant = tenantCommonRepository.findById(tenantId); + if (tenant.isPresent()) { + PartnerCallBackType partnerCallBackType = tenant.get().getStatus() == TenantFileStatus.VALIDATED ? + PartnerCallBackType.VERIFIED_ACCOUNT : + PartnerCallBackType.CREATED_ACCOUNT; + partnerCallBackService.sendCallBack(tenant.get(), partnerCallBackType); + } + } +} \ No newline at end of file diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentToProcessTask.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentToProcessTask.java deleted file mode 100644 index b45cfb5f4..000000000 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/documenttoprocess/DocumentToProcessTask.java +++ /dev/null @@ -1,56 +0,0 @@ -package fr.dossierfacile.scheduler.tasks.documenttoprocess; - -import com.google.gson.Gson; -import fr.dossierfacile.common.entity.Document; -import fr.dossierfacile.scheduler.LoggingContext; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.amqp.core.AmqpTemplate; -import org.springframework.amqp.core.Message; -import org.springframework.amqp.core.MessageProperties; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.time.LocalDateTime; -import java.util.Collections; -import java.util.concurrent.TimeUnit; - -import static fr.dossierfacile.scheduler.tasks.TaskName.PDF_GENERATION; - -@Service -@Slf4j -@RequiredArgsConstructor -public class DocumentToProcessTask { - - private final AmqpTemplate amqpTemplate; - private final Gson gson; - private final DocumentRepository documentRepository; - - @Value("${rabbitmq.exchange.pdf.generator}") - private String exchangePdfGenerator; - @Value("${rabbitmq.routing.key.pdf.generator.watermark-document}") - private String routingKeyPdfGenerator; - - @Scheduled(cron = "${cron.process.pdf.generation.failed}") - public void launchFailedPDFGeneration() { - LoggingContext.startTask(PDF_GENERATION); - LocalDateTime now = LocalDateTime.now(); - LocalDateTime toDateTime = now.minusMinutes(30); - documentRepository.findToProcessWithoutPDFToDate(toDateTime).forEach(this::sendForPDFGeneration); - LoggingContext.endTask(); - } - - private void sendForPDFGeneration(Document document) { - log.debug("Sending document with ID [{}] for pdf generation", document.getId()); - amqpTemplate.send(exchangePdfGenerator, routingKeyPdfGenerator, messageWithId(document.getId())); - } - - private Message messageWithId(Long id) { - MessageProperties properties = new MessageProperties(); - properties.setHeader("timestamp", System.currentTimeMillis()); - return new Message( - gson.toJson(Collections.singletonMap("id", String.valueOf(id))).getBytes(), - properties); - } -} diff --git a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/BackupFilesTask.java b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/BackupFilesTask.java index c8d042352..f1a260c21 100644 --- a/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/BackupFilesTask.java +++ b/dossierfacile-task-scheduler/src/main/java/fr/dossierfacile/scheduler/tasks/storagesynchronization/BackupFilesTask.java @@ -24,7 +24,7 @@ public class BackupFilesTask { private final StorageFileRepository storageFileRepository; private final FileStorageService fileStorageService; - @Scheduled(fixedDelayString = "${scheduled.process.storage.backup.delay.ms}") + @Scheduled(fixedDelayString = "${scheduled.process.storage.backup.delay.ms}", initialDelayString = "${scheduled.process.storage.backup.delay.ms}") public void scheduleBackupTask() { LoggingContext.startTask(STORAGE_FILES_BACKUP); Pageable limit = PageRequest.of(0, 100); diff --git a/dossierfacile-task-scheduler/src/main/resources/application.properties b/dossierfacile-task-scheduler/src/main/resources/application.properties index 52a4bda64..163987a46 100644 --- a/dossierfacile-task-scheduler/src/main/resources/application.properties +++ b/dossierfacile-task-scheduler/src/main/resources/application.properties @@ -37,15 +37,19 @@ sendinblue.apikey= sendinblue.template.id.first.warning.for.deletion.of.documents= sendinblue.template.id.second.warning.for.deletion.of.documents= sendinblue.url.domain= +sendinblue.template.id.deleted.document.with.failed.pdf=106 # scheduled process cron.process.warnings= cron.account-deletion= -cron.process.pdf.generation.failed=0 0 3,6,9,13,17,20,23 * * * +cron.process.pdf.generation.failed=0 30 1,7,12,19 * * * +cron.delete.document.with.failed.pdf=0 0 6,22 * * * scheduled.process.storage.backup.delay.ms=10000 scheduled.process.storage.delete.delay.ms=10000 garbage-collection.seconds-between-iterations=60 +document.pdf.failed.delay.before.delete.hours=480 + # Logging logging.config=classpath:logback-spring-delayed.xml logging.level.root=info From 3c96f955eb6c33f6d5c89c5e4a4e48d81a2015fd Mon Sep 17 00:00:00 2001 From: Fabien Date: Fri, 22 Dec 2023 15:57:42 +0100 Subject: [PATCH 15/19] feat: update validation rules --- .../common/entity/ocr/TaxIncomeMainFile.java | 4 +- .../enums/ParsedFileClassification.java | 2 +- dossierfacile-process-file/pom.xml | 6 +- .../file/amqp/AnalyzeDocumentReceiver.java | 4 +- .../file/service/AnalyzeDocumentService.java | 14 ++- ...DocumentRulesValidationServiceFactory.java | 18 +--- ...aranteeProviderRulesValidationService.java | 9 +- .../IncomeTaxRulesValidationService.java | 47 +++++----- .../documentrules/RulesValidationService.java | 3 +- .../file/service/ocr/TaxIncomeParser.java | 6 +- .../IncomeTaxRulesValidationServiceTest.java | 94 +++++++++++++++++++ 11 files changed, 156 insertions(+), 51 deletions(-) create mode 100644 dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationServiceTest.java diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ocr/TaxIncomeMainFile.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ocr/TaxIncomeMainFile.java index 58abb3113..8c751131b 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ocr/TaxIncomeMainFile.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ocr/TaxIncomeMainFile.java @@ -24,9 +24,9 @@ public class TaxIncomeMainFile implements ParsedFile, Serializable { String declarant2NumFiscal; String declarant2Nom; String nombreDeParts; - String anneeDesRevenus; + Integer anneeDesRevenus; String dateDeMiseEnRecouvrement; - String revenuFiscalDeReference; + Integer revenuFiscalDeReference; String numeroFiscalDeclarant1; String numeroFiscalDeclarant2; String referenceAvis; diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/ParsedFileClassification.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/ParsedFileClassification.java index b36856e57..fa390578a 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/ParsedFileClassification.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/ParsedFileClassification.java @@ -12,7 +12,7 @@ public enum ParsedFileClassification { TAX_INCOME(TaxIncomeMainFile.class), TAX_INCOME_LEAF(TaxIncomeLeaf.class), - GUARANTEE_PROVIDER(GuaranteeProviderFile .class); + GUARANTEE_PROVIDER(GuaranteeProviderFile.class); Class classificationClass; } diff --git a/dossierfacile-process-file/pom.xml b/dossierfacile-process-file/pom.xml index f62aba52c..96e5b5cfe 100644 --- a/dossierfacile-process-file/pom.xml +++ b/dossierfacile-process-file/pom.xml @@ -75,17 +75,17 @@ com.fasterxml.jackson.core jackson-databind - 2.15.2 + 2.13.5 com.fasterxml.jackson.core jackson-core - 2.15.2 + 2.13.5 com.fasterxml.jackson.core jackson-annotations - 2.13.4 + 2.13.5 com.google.zxing diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/amqp/AnalyzeDocumentReceiver.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/amqp/AnalyzeDocumentReceiver.java index 17b85b2e9..748fff8a6 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/amqp/AnalyzeDocumentReceiver.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/amqp/AnalyzeDocumentReceiver.java @@ -2,7 +2,6 @@ import com.google.gson.Gson; import fr.dossierfacile.process.file.service.AnalyzeDocumentService; - import fr.dossierfacile.process.file.service.interfaces.DocumentService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -22,9 +21,11 @@ public class AnalyzeDocumentReceiver { private final DocumentService documentService; @Value("${rabbitmq.document.analyze.delay}") private Long documentAnalysisDelay; + @RabbitListener(queues = "${rabbitmq.queue.document.analyze}", containerFactory = "retryContainerFactory") public void receiveDocument(Message message) { log.debug("Received message on queue.document.analyze to process:" + message); + Long msgTimestamp = message.getMessageProperties().getHeader("timestamp"); delayExecution(msgTimestamp); @@ -37,6 +38,7 @@ public void receiveDocument(Message message) { } else { log.debug("Ignore document analysis because document is NOT up to date"); } + } private void delayExecution(Long msgTimestamp) { diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeDocumentService.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeDocumentService.java index 4f72faad2..56f1c35ff 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeDocumentService.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeDocumentService.java @@ -10,6 +10,10 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import java.util.LinkedList; +import java.util.List; @Slf4j @Service @@ -27,9 +31,13 @@ public void processDocument(Long documentId) { log.info("Ignoring document {} because it has already been analysed", documentId); } try { - RulesValidationService rulesValidationService = documentRulesValidationServiceFactory.get(document); - if (rulesValidationService != null) { - DocumentAnalysisReport report = rulesValidationService.process(document); + List rulesValidationServices = documentRulesValidationServiceFactory.getServices(document); + if (!CollectionUtils.isEmpty(rulesValidationServices )) { + DocumentAnalysisReport report = DocumentAnalysisReport.builder() + .document(document) + .brokenRules(new LinkedList<>()) + .build(); + rulesValidationServices.stream().forEach( rulesService -> rulesService.process(document, report) ); document.setDocumentAnalysisReport(report); documentAnalysisReportRepository.save(report); documentRepository.save(document); diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/DocumentRulesValidationServiceFactory.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/DocumentRulesValidationServiceFactory.java index ba6a7a6fe..e4e26f9e5 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/DocumentRulesValidationServiceFactory.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/DocumentRulesValidationServiceFactory.java @@ -1,26 +1,18 @@ package fr.dossierfacile.process.file.service.documentrules; import fr.dossierfacile.common.entity.Document; -import fr.dossierfacile.common.enums.DocumentCategory; import lombok.AllArgsConstructor; import org.springframework.stereotype.Service; -import static fr.dossierfacile.common.enums.DocumentSubCategory.CERTIFICATE_VISA; +import java.util.List; +import java.util.stream.Collectors; @Service @AllArgsConstructor public class DocumentRulesValidationServiceFactory { - private final IncomeTaxRulesValidationService incomeTaxRulesValidationService; - private final GuaranteeProviderRulesValidationService guaranteeProviderRulesValidationService; + private List rulesValidationServiceList; - public RulesValidationService get(Document document) { - if (document.getDocumentCategory() == DocumentCategory.TAX) { - return incomeTaxRulesValidationService; - } - if (document.getDocumentCategory() == DocumentCategory.IDENTIFICATION - && document.getDocumentSubCategory() == CERTIFICATE_VISA) { - return guaranteeProviderRulesValidationService; - } - return null; + public List getServices(Document document) { + return rulesValidationServiceList.stream().filter(rvs -> rvs.shouldBeApplied(document)).collect(Collectors.toList()); } } \ No newline at end of file diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/GuaranteeProviderRulesValidationService.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/GuaranteeProviderRulesValidationService.java index db14a9f87..68ccfb4c4 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/GuaranteeProviderRulesValidationService.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/GuaranteeProviderRulesValidationService.java @@ -3,6 +3,8 @@ import fr.dossierfacile.common.entity.Document; import fr.dossierfacile.common.entity.DocumentAnalysisReport; import fr.dossierfacile.common.entity.DocumentAnalysisStatus; +import fr.dossierfacile.common.enums.DocumentCategory; +import fr.dossierfacile.common.enums.DocumentSubCategory; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -12,7 +14,12 @@ @Slf4j public class GuaranteeProviderRulesValidationService implements RulesValidationService { @Override - public DocumentAnalysisReport process(Document document) { + public boolean shouldBeApplied(Document document){ + return document.getDocumentCategory() == DocumentCategory.IDENTIFICATION + && document.getDocumentSubCategory() == DocumentSubCategory.CERTIFICATE_VISA; + } + @Override + public DocumentAnalysisReport process(Document document, DocumentAnalysisReport report) { log.debug("TODO"); // TODO Currently there is not implemented rules return DocumentAnalysisReport.builder().analysisStatus(DocumentAnalysisStatus.UNDEFINED).document(document).build(); diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java index c74e74fe2..cdf85b64e 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java @@ -1,17 +1,10 @@ package fr.dossierfacile.process.file.service.documentrules; -import fr.dossierfacile.common.entity.BarCodeDocumentType; -import fr.dossierfacile.common.entity.BarCodeFileAnalysis; -import fr.dossierfacile.common.entity.Document; -import fr.dossierfacile.common.entity.DocumentAnalysisReport; -import fr.dossierfacile.common.entity.DocumentAnalysisStatus; -import fr.dossierfacile.common.entity.DocumentBrokenRule; -import fr.dossierfacile.common.entity.DocumentRule; -import fr.dossierfacile.common.entity.File; -import fr.dossierfacile.common.entity.ParsedFileAnalysis; -import fr.dossierfacile.common.entity.Person; +import fr.dossierfacile.common.entity.*; import fr.dossierfacile.common.entity.ocr.TaxIncomeLeaf; import fr.dossierfacile.common.entity.ocr.TaxIncomeMainFile; +import fr.dossierfacile.common.enums.DocumentCategory; +import fr.dossierfacile.common.enums.DocumentSubCategory; import fr.dossierfacile.common.enums.ParsedFileAnalysisStatus; import fr.dossierfacile.common.enums.ParsedFileClassification; import fr.dossierfacile.process.file.barcode.twoddoc.parsing.TwoDDocDataType; @@ -21,10 +14,10 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; +import javax.validation.constraints.NotNull; import java.text.Normalizer; import java.time.LocalDate; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -33,6 +26,11 @@ @RequiredArgsConstructor @Slf4j public class IncomeTaxRulesValidationService implements RulesValidationService { + @Override + public boolean shouldBeApplied(Document document) { + return document.getDocumentCategory() == DocumentCategory.TAX + && document.getDocumentSubCategory() == DocumentSubCategory.MY_NAME; + } private TaxIncomeMainFile fromQR(BarCodeFileAnalysis barCodeFileAnalysis) { Map dataWithLabel = (Map) barCodeFileAnalysis.getVerifiedData(); @@ -41,10 +39,10 @@ private TaxIncomeMainFile fromQR(BarCodeFileAnalysis barCodeFileAnalysis) { .declarant1Nom(dataWithLabel.get(TwoDDocDataType.ID_46.getLabel())) .declarant2NumFiscal(dataWithLabel.get(TwoDDocDataType.ID_49.getLabel())) .declarant2Nom(dataWithLabel.get(TwoDDocDataType.ID_48.getLabel())) - .anneeDesRevenus(dataWithLabel.get(TwoDDocDataType.ID_45.getLabel())) + .anneeDesRevenus(Integer.parseInt(dataWithLabel.get(TwoDDocDataType.ID_45.getLabel()))) .nombreDeParts(dataWithLabel.get(TwoDDocDataType.ID_43.getLabel())) .dateDeMiseEnRecouvrement(dataWithLabel.get(TwoDDocDataType.ID_4A.getLabel())) - .revenuFiscalDeReference(dataWithLabel.get(TwoDDocDataType.ID_41.getLabel())) + .revenuFiscalDeReference(Integer.parseInt(dataWithLabel.get(TwoDDocDataType.ID_41.getLabel()))) .numeroFiscalDeclarant1(dataWithLabel.get(TwoDDocDataType.ID_47.getLabel())) .numeroFiscalDeclarant2(dataWithLabel.get(TwoDDocDataType.ID_49.getLabel())) .referenceAvis(dataWithLabel.get(TwoDDocDataType.ID_44.getLabel())).build(); @@ -60,13 +58,8 @@ private static String normalizeName(String name) { @Override @Transactional - public DocumentAnalysisReport process(Document document) { - DocumentAnalysisReport report = DocumentAnalysisReport.builder().document(document).build(); - List brokenRules = Optional.ofNullable(report.getBrokenRules()) - .orElseGet(() -> { - report.setBrokenRules(new LinkedList<>()); - return report.getBrokenRules(); - }); + public DocumentAnalysisReport process(Document document, @NotNull DocumentAnalysisReport report) { + List brokenRules = report.getBrokenRules(); // Parse Rule for (File dfFile : document.getFiles()) { @@ -103,9 +96,9 @@ public DocumentAnalysisReport process(Document document) { && parsedDocument.getAnneeDesRevenus() != null && parsedDocument.getDeclarant1Nom() != null && parsedDocument.getRevenuFiscalDeReference() != null - && (!qrDocument.getAnneeDesRevenus().equalsIgnoreCase(parsedDocument.getAnneeDesRevenus()) + && (!qrDocument.getAnneeDesRevenus().equals(parsedDocument.getAnneeDesRevenus()) || !qrDocument.getDeclarant1Nom().equalsIgnoreCase(parsedDocument.getDeclarant1Nom()) - || !qrDocument.getRevenuFiscalDeReference().equalsIgnoreCase(parsedDocument.getRevenuFiscalDeReference().replaceAll("\\s", "")) + || !qrDocument.getRevenuFiscalDeReference().equals(parsedDocument.getRevenuFiscalDeReference()) )) { log.error("Le 2DDoc code ne correspond pas au contenu du document tenantId:" + document.getTenant().getId()); brokenRules.add(DocumentBrokenRule.builder() @@ -122,7 +115,7 @@ public DocumentAnalysisReport process(Document document) { if (dfFile.getFileAnalysis() != null) { TaxIncomeMainFile qrDocument = fromQR(dfFile.getFileAnalysis()); if (qrDocument.getAnneeDesRevenus() != null) - providedYears.add(Integer.parseInt(qrDocument.getAnneeDesRevenus())); + providedYears.add(qrDocument.getAnneeDesRevenus()); } } Integer maxYear = providedYears.stream().max(Integer::compare).orElse(null); @@ -221,4 +214,12 @@ && normalizeName(qrDocument.getDeclarant2Nom()).contains(normalizeName(lastName) return report; } + + + private boolean compareName(String incomeTaxFullName, String firstName, String lastName) { + String fullName = normalizeName(incomeTaxFullName); + String givenFullName = normalizeName(normalizeName(lastName) + " " + normalizeName(firstName)); + + return false; + } } \ No newline at end of file diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/RulesValidationService.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/RulesValidationService.java index f6abc0820..7a2b03f7d 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/RulesValidationService.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/RulesValidationService.java @@ -4,5 +4,6 @@ import fr.dossierfacile.common.entity.DocumentAnalysisReport; public interface RulesValidationService { - DocumentAnalysisReport process(Document document); + boolean shouldBeApplied(Document document); + DocumentAnalysisReport process(Document document, DocumentAnalysisReport report); } \ No newline at end of file diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/ocr/TaxIncomeParser.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/ocr/TaxIncomeParser.java index 872d0483a..b6ea288a4 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/ocr/TaxIncomeParser.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/ocr/TaxIncomeParser.java @@ -84,7 +84,7 @@ public TaxIncomeMainFile parse(File file) { String zoneTitle = tesseract.doOCR(image, TaxIncomeZones.scale(zones.zoneTitle, scale)); Matcher matcher = incomeYearPattern.matcher(zoneTitle); if (matcher.find()) { - result.setAnneeDesRevenus(matcher.group(1)); + result.setAnneeDesRevenus(Integer.parseInt(matcher.group(1).replaceAll("\\s", ""))); } else { log.warn("Income year not found"); continue; @@ -114,14 +114,14 @@ public TaxIncomeMainFile parse(File file) { String zoneRevenuPart = tesseract.doOCR(image, TaxIncomeZones.scale(zones.zoneRevenuPart, scale)); Matcher partCountMatcher = partCountPattern.matcher(zoneRevenuPart); if (partCountMatcher.find()) { - result.setNombreDeParts(partCountMatcher.group(1)); + result.setNombreDeParts(partCountMatcher.group(1).replaceAll("\\s", "")); } else { log.warn("\"Nombre de parts\" not found \n" + zoneRevenuPart); break; } Matcher incomeAmountMatcher = incomeAmountPattern.matcher(zoneRevenuPart); if (incomeAmountMatcher.find()) { - result.setRevenuFiscalDeReference(incomeAmountMatcher.group(1)); + result.setRevenuFiscalDeReference(Integer.parseInt(incomeAmountMatcher.group(1).replaceAll("\\s", ""))); break; } else { log.warn("\"Revenu fiscal de référence\" not found" + zoneRevenuPart); diff --git a/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationServiceTest.java b/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationServiceTest.java new file mode 100644 index 000000000..2b33e047f --- /dev/null +++ b/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationServiceTest.java @@ -0,0 +1,94 @@ +package fr.dossierfacile.process.file.service.documentrules; + +import com.fasterxml.jackson.databind.ObjectMapper; +import fr.dossierfacile.common.entity.*; +import fr.dossierfacile.common.enums.DocumentCategory; +import fr.dossierfacile.common.enums.DocumentSubCategory; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.LinkedList; + +class IncomeTaxRulesValidationServiceTest { + + + private IncomeTaxRulesValidationService incomeTaxRulesValidationService = new IncomeTaxRulesValidationService(); + + + private Document buildValidTaxDocument() throws Exception { + Tenant tenant = Tenant.builder() + .firstName("Jean") + .lastName("DUPONT") + .build(); + BarCodeFileAnalysis barCodeFileAnalysis = BarCodeFileAnalysis.builder() + .verifiedData( + new ObjectMapper().readValue(""" + { + "Déclarant 1": "DUPONT Jean", + "Déclarant 2": "DUPONT Marie", + "Nombre de parts": "2", + "Année des revenus": "2022", + "Date de mise en recouvrement": "30092023", + "Revenu fiscal de référence": "42902", + "Numéro fiscal du déclarant 1": "1234567890123", + "Numéro fiscal du déclarant 2": "1234567890124", + "Référence d’avis d’impôt": "2310A12345678" + } + """, Object.class) + ) + .build(); + File dfFile = File.builder() + .fileAnalysis(barCodeFileAnalysis) + .build(); + return Document.builder() + .tenant(tenant) + .documentCategory(DocumentCategory.TAX) + .documentSubCategory(DocumentSubCategory.MY_NAME) + .files(Arrays.asList(dfFile)) + .noDocument(true) + .build(); + } + + @Test + public void document_full_test() throws Exception { + Document document = buildValidTaxDocument(); + DocumentAnalysisReport report = DocumentAnalysisReport.builder() + .document(document) + .brokenRules(new LinkedList<>()) + .build(); + incomeTaxRulesValidationService.process(document, report); + + Assertions.assertThat(report.getAnalysisStatus()).isEqualTo(DocumentAnalysisStatus.CHECKED); + } +/* + @Test + public void compare_name_should_be_ok() { + String givenLastName = "Jean Phillipe"; + String givenFirstName = "DE LA MARCHE"; + String fullName = "DE LA MARCHE PICQUET Edouard Jean Phillipe"; + } + + @Test + public void compare_name_should_be_ko() { + String givenLastName = "Jean Phillipe"; + String givenFirstName = "DE LA MARCHE"; + String fullName = "PICQUET Edouard Jean Phillipe"; + } + + @Test + public void compare_name_should_be_ok2() { + String givenLastName = "Jean Phillipe"; + String givenFirstName = "MARCHE"; + String fullName = "ROSE-MARCHE Jean"; + } + + @Test + public void compare_name_should_be_ok_basc() { + String givenLastName = "Jean"; + String givenFirstName = "MARCHE"; + String fullName = "MARCHE Jean"; + } + + */ +} \ No newline at end of file From 420017aab24f7ba4f8215a14a24761bf4ec61230 Mon Sep 17 00:00:00 2001 From: Fabien Date: Tue, 2 Jan 2024 18:37:20 +0100 Subject: [PATCH 16/19] feat: update validation rules - names --- .../process/file/service/AnalysisContext.java | 16 ------ .../process/file/service/AnalyzeFile.java | 31 ++-------- ...sor.java => StorageFileLoaderService.java} | 32 +++++------ .../IncomeTaxRulesValidationService.java | 26 ++------- .../processors/BarCodeFileProcessor.java | 15 ++--- .../processors/OcrParserFileProcessor.java | 56 +++++++++++-------- .../file/service/processors/Processor.java | 10 +--- .../util/FileParsingEligibilityCriteria.java | 25 --------- .../file/util/PersonNameComparator.java | 44 +++++++++++++++ .../IncomeTaxRulesValidationServiceTest.java | 39 ++++--------- .../file/util/PersonNameComparatorTest.java | 46 +++++++++++++++ 11 files changed, 168 insertions(+), 172 deletions(-) delete mode 100644 dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalysisContext.java rename dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/{processors/LoadFileProcessor.java => StorageFileLoaderService.java} (51%) delete mode 100644 dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/FileParsingEligibilityCriteria.java create mode 100644 dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/PersonNameComparator.java create mode 100644 dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/util/PersonNameComparatorTest.java diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalysisContext.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalysisContext.java deleted file mode 100644 index 4eb2db9f9..000000000 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalysisContext.java +++ /dev/null @@ -1,16 +0,0 @@ -package fr.dossierfacile.process.file.service; - -import fr.dossierfacile.common.entity.BarCodeFileAnalysis; -import fr.dossierfacile.common.entity.File; -import fr.dossierfacile.common.entity.ocr.ParsedFile; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -public class AnalysisContext { - private File dfFile; - private BarCodeFileAnalysis barCodeFileAnalysis; - private java.io.File file; - private ParsedFile parsedDocument; -} diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeFile.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeFile.java index 84aaba962..2ad7c162e 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeFile.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeFile.java @@ -1,48 +1,27 @@ package fr.dossierfacile.process.file.service; -import fr.dossierfacile.common.entity.File; import fr.dossierfacile.process.file.repository.FileRepository; import fr.dossierfacile.process.file.service.processors.BarCodeFileProcessor; -import fr.dossierfacile.process.file.service.processors.LoadFileProcessor; import fr.dossierfacile.process.file.service.processors.OcrParserFileProcessor; -import fr.dossierfacile.process.file.util.FileParsingEligibilityCriteria; -import fr.dossierfacile.process.file.util.QrCodeFileAnalysisCriteria; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import java.util.Objects; import java.util.Optional; -import static fr.dossierfacile.common.enums.DocumentSubCategory.MY_NAME; - @Slf4j @Service @AllArgsConstructor public class AnalyzeFile { - - private final LoadFileProcessor loadFileProcessor; private final BarCodeFileProcessor barCodeFileProcessor; private final OcrParserFileProcessor ocrParserFileProcessor; private final FileRepository fileRepository; public void processFile(Long fileId) { - AnalysisContext context = new AnalysisContext(); - - Optional.of(context) - .map(ctx -> { - ctx.setDfFile(fileRepository.findById(fileId).orElse(null)); - return ctx; - }) - .filter(ctx -> QrCodeFileAnalysisCriteria.shouldBeAnalyzed(ctx.getDfFile())) - .map(barCodeFileProcessor::process); - - Optional.of(context) - .filter(ctx -> FileParsingEligibilityCriteria.shouldBeParse(ctx.getDfFile())) - .map(loadFileProcessor::process) - .map(ocrParserFileProcessor::process) - .map(loadFileProcessor::cleanContext); + Optional.ofNullable(fileRepository.findById(fileId).orElse(null)) + .filter(Objects::nonNull) + .map(barCodeFileProcessor::process) + .map(ocrParserFileProcessor::process); } - - - } diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/LoadFileProcessor.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/StorageFileLoaderService.java similarity index 51% rename from dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/LoadFileProcessor.java rename to dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/StorageFileLoaderService.java index a362af186..b2bada098 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/LoadFileProcessor.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/StorageFileLoaderService.java @@ -1,12 +1,13 @@ -package fr.dossierfacile.process.file.service.processors; +package fr.dossierfacile.process.file.service; +import fr.dossierfacile.common.entity.StorageFile; import fr.dossierfacile.common.service.interfaces.FileStorageService; -import fr.dossierfacile.process.file.service.AnalysisContext; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.MediaType; import org.springframework.stereotype.Service; +import java.io.File; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -16,36 +17,33 @@ @Service @AllArgsConstructor @Slf4j -public class LoadFileProcessor implements Processor { +public class StorageFileLoaderService { private FileStorageService fileStorageService; - public AnalysisContext process(AnalysisContext analysisContext) { + public File getTemporaryFilePath(StorageFile storageFile) { try { - try (InputStream in = fileStorageService.download(analysisContext.getDfFile().getStorageFile())) { - Path temporaryFile = Files.createTempFile("tax-" + analysisContext.getDfFile().getStorageFile().getId() + try (InputStream in = fileStorageService.download(storageFile)) { + Path temporaryFile = Files.createTempFile("tax-" + storageFile.getId() + "-" + UUID.randomUUID(), - MediaType.APPLICATION_PDF_VALUE.equalsIgnoreCase(analysisContext.getDfFile().getStorageFile().getContentType()) ? ".pdf" : ""); + MediaType.APPLICATION_PDF_VALUE.equalsIgnoreCase(storageFile.getContentType()) ? ".pdf" : ""); // actually we only need first file - Files.copy(in, temporaryFile, StandardCopyOption.REPLACE_EXISTING); - - analysisContext.setFile(temporaryFile.toFile()); + if (Files.copy(in, temporaryFile, StandardCopyOption.REPLACE_EXISTING) > 0) { + return temporaryFile.toFile(); + } } } catch (Exception e) { log.error("Cannot read and save files from document"); } - return analysisContext; + return null; } - @Override - public AnalysisContext cleanContext(AnalysisContext analysisContext) { - if (analysisContext.getFile() != null) { + public void removeFileIfExist(File file) { + if (file != null) { try { - analysisContext.getFile().delete(); + file.delete(); } catch (Exception e) { log.warn("Cannot clean up this file", e); } - analysisContext.setFile(null); } - return analysisContext; } } diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java index cdf85b64e..c5ea22bb5 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java @@ -8,6 +8,7 @@ import fr.dossierfacile.common.enums.ParsedFileAnalysisStatus; import fr.dossierfacile.common.enums.ParsedFileClassification; import fr.dossierfacile.process.file.barcode.twoddoc.parsing.TwoDDocDataType; +import fr.dossierfacile.process.file.util.PersonNameComparator; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -15,7 +16,6 @@ import org.springframework.util.CollectionUtils; import javax.validation.constraints.NotNull; -import java.text.Normalizer; import java.time.LocalDate; import java.util.ArrayList; import java.util.List; @@ -48,14 +48,6 @@ private TaxIncomeMainFile fromQR(BarCodeFileAnalysis barCodeFileAnalysis) { .referenceAvis(dataWithLabel.get(TwoDDocDataType.ID_44.getLabel())).build(); } - private static String normalizeName(String name) { - if (name == null) - return null; - String normalized = Normalizer.normalize(name, Normalizer.Form.NFD); - return normalized.replace('-', ' ') - .replaceAll("[\\p{InCombiningDiacriticalMarks}]", "").toUpperCase().trim(); - } - @Override @Transactional public DocumentAnalysisReport process(Document document, @NotNull DocumentAnalysisReport report) { @@ -160,12 +152,10 @@ public DocumentAnalysisReport process(Document document, @NotNull DocumentAnalys if (analysis.getDocumentType() == BarCodeDocumentType.TAX_ASSESSMENT) { TaxIncomeMainFile qrDocument = fromQR(dfFile.getFileAnalysis()); - if (!(normalizeName(qrDocument.getDeclarant1Nom()).contains(normalizeName(firstName)) - && normalizeName(qrDocument.getDeclarant1Nom()).contains(normalizeName(lastName)) + if (!(PersonNameComparator.bearlyEqualsTo(qrDocument.getDeclarant1Nom(), lastName, firstName) || (qrDocument.getDeclarant2Nom() != null && - normalizeName(qrDocument.getDeclarant2Nom()).contains(normalizeName(firstName)) - && normalizeName(qrDocument.getDeclarant2Nom()).contains(normalizeName(lastName)) - ))) { + PersonNameComparator.bearlyEqualsTo(qrDocument.getDeclarant1Nom(), lastName, firstName)) + )) { log.error("Le nom/prenom ne correpond pas à l'uilitsation tenantId:" + document.getTenant().getId() + " firstname: " + firstName); brokenRules.add(DocumentBrokenRule.builder() .rule(DocumentRule.R_TAX_NAMES) @@ -214,12 +204,4 @@ && normalizeName(qrDocument.getDeclarant2Nom()).contains(normalizeName(lastName) return report; } - - - private boolean compareName(String incomeTaxFullName, String firstName, String lastName) { - String fullName = normalizeName(incomeTaxFullName); - String givenFullName = normalizeName(normalizeName(lastName) + " " + normalizeName(firstName)); - - return false; - } } \ No newline at end of file diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/BarCodeFileProcessor.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/BarCodeFileProcessor.java index 9dec06907..eae70d583 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/BarCodeFileProcessor.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/BarCodeFileProcessor.java @@ -5,10 +5,10 @@ import fr.dossierfacile.common.service.interfaces.FileStorageService; import fr.dossierfacile.process.file.barcode.InMemoryFile; import fr.dossierfacile.process.file.repository.BarCodeFileAnalysisRepository; -import fr.dossierfacile.process.file.service.AnalysisContext; import fr.dossierfacile.process.file.service.qrcodeanalysis.DocumentClassifier; import fr.dossierfacile.process.file.service.qrcodeanalysis.QrCodeFileAuthenticator; import fr.dossierfacile.process.file.service.qrcodeanalysis.TwoDDocFileAuthenticator; +import fr.dossierfacile.process.file.util.QrCodeFileAnalysisCriteria; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -26,18 +26,15 @@ public class BarCodeFileProcessor implements Processor { private final BarCodeFileAnalysisRepository analysisRepository; private final FileStorageService fileStorageService; - public AnalysisContext process(AnalysisContext context) { - File file = context.getDfFile(); - if (analysisRepository.hasNotAlreadyBeenAnalyzed(file)) { + public File process(File file) { + if (QrCodeFileAnalysisCriteria.shouldBeAnalyzed(file) && + analysisRepository.hasNotAlreadyBeenAnalyzed(file)) { long start = System.currentTimeMillis(); log.info("Starting analysis of file"); - context.setBarCodeFileAnalysis( - downloadAndAnalyze(file) - .map(analysis -> save(file, analysis)) - .orElse(null)); + downloadAndAnalyze(file).map(analysis -> save(file, analysis)); log.info("Analysis of file finished in {} ms", System.currentTimeMillis() - start); } - return context; + return file; } private Optional downloadAndAnalyze(File file) { diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/OcrParserFileProcessor.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/OcrParserFileProcessor.java index b3911332c..a167f27b9 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/OcrParserFileProcessor.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/OcrParserFileProcessor.java @@ -7,11 +7,11 @@ import fr.dossierfacile.common.enums.ParsedFileAnalysisStatus; import fr.dossierfacile.common.repository.ParsedFileAnalysisRepository; import fr.dossierfacile.process.file.repository.FileRepository; -import fr.dossierfacile.process.file.service.AnalysisContext; import fr.dossierfacile.process.file.service.ocr.GuaranteeVisaleParser; import fr.dossierfacile.process.file.service.ocr.OcrParser; import fr.dossierfacile.process.file.service.ocr.TaxIncomeLeafParser; import fr.dossierfacile.process.file.service.ocr.TaxIncomeParser; +import fr.dossierfacile.process.file.service.StorageFileLoaderService; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -27,6 +27,7 @@ @Service @AllArgsConstructor public class OcrParserFileProcessor implements Processor { + private StorageFileLoaderService storageFileLoaderService; private final FileRepository fileRepository; private final ParsedFileAnalysisRepository parsedFileAnalysisRepository; private final TaxIncomeParser taxIncomeParser; @@ -46,31 +47,42 @@ private List getParsers(File file) { return null; } - public AnalysisContext process(AnalysisContext context) { - List parsers = getParsers(context.getDfFile()); + public File process(File dfFile) { + + List parsers = getParsers(dfFile); if (CollectionUtils.isEmpty(parsers)) { - log.error("There is not parser associateed to this kind of document - configuration error"); - return context; + log.debug("There is not parser associateed to this kind of document"); + return dfFile; + } + + java.io.File file = storageFileLoaderService.getTemporaryFilePath(dfFile.getStorageFile()); + if (file == null) { + log.error("File reading Error"); + return dfFile; } - for (OcrParser parser : parsers) { - try { - ParsedFile parsedDocument = parser.parse(context.getFile()); - ParsedFileAnalysis parsedFileAnalysis = ParsedFileAnalysis.builder() - .analysisStatus(ParsedFileAnalysisStatus.COMPLETED) - .parsedFile(parsedDocument) - .classification(parsedDocument.getClassification()) - .build(); + try { + for (OcrParser parser : parsers) { + try { + ParsedFile parsedDocument = parser.parse(file); + ParsedFileAnalysis parsedFileAnalysis = ParsedFileAnalysis.builder() + .analysisStatus(ParsedFileAnalysisStatus.COMPLETED) + .parsedFile(parsedDocument) + .classification(parsedDocument.getClassification()) + .build(); - parsedFileAnalysis.setFile(context.getDfFile()); - parsedFileAnalysisRepository.save(parsedFileAnalysis); - context.getDfFile().setParsedFileAnalysis(parsedFileAnalysis); - fileRepository.save(context.getDfFile()); - log.info("Successfully parse file {}", context.getDfFile().getId()); - break; - } catch (Exception e) { - log.warn("Unable to parse file {}", context.getDfFile().getId()); + parsedFileAnalysis.setFile(dfFile); + parsedFileAnalysisRepository.save(parsedFileAnalysis); + dfFile.setParsedFileAnalysis(parsedFileAnalysis); + fileRepository.save(dfFile); + log.info("Successfully parse file {}", dfFile.getId()); + break; + } catch (Exception e) { + log.warn("Unable to parse file {}", dfFile.getId()); + } } + } finally { + storageFileLoaderService.removeFileIfExist(file); } - return context; + return dfFile; } } diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/Processor.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/Processor.java index b862e5e3b..4e034dbbd 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/Processor.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/processors/Processor.java @@ -1,13 +1,7 @@ package fr.dossierfacile.process.file.service.processors; -import fr.dossierfacile.process.file.service.AnalysisContext; +import fr.dossierfacile.common.entity.File; public interface Processor { - AnalysisContext process(AnalysisContext context); - /** - * (Optional) Clean up the elements that were added to the context by this process. - */ - default AnalysisContext cleanContext(AnalysisContext context) { - return context; - } + File process(File dfFile); } diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/FileParsingEligibilityCriteria.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/FileParsingEligibilityCriteria.java deleted file mode 100644 index 1386e69d2..000000000 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/FileParsingEligibilityCriteria.java +++ /dev/null @@ -1,25 +0,0 @@ -package fr.dossierfacile.process.file.util; - -import fr.dossierfacile.common.entity.Document; -import fr.dossierfacile.common.entity.File; -import lombok.extern.slf4j.Slf4j; - -import static fr.dossierfacile.common.enums.DocumentSubCategory.CERTIFICATE_VISA; -import static fr.dossierfacile.common.enums.DocumentSubCategory.MY_NAME; - -@Slf4j -public class FileParsingEligibilityCriteria { - - public static boolean shouldBeParse(File file) { - if (file == null) { - log.error("File cannot be empty dfFile"); - return false; - } - Document document = file.getDocument(); - return switch (document.getDocumentCategory()) { - case TAX -> file.getDocument().getDocumentSubCategory() == MY_NAME; - case IDENTIFICATION -> file.getDocument().getDocumentSubCategory() == CERTIFICATE_VISA; - default -> false; - }; - } -} diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/PersonNameComparator.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/PersonNameComparator.java new file mode 100644 index 000000000..23db4232d --- /dev/null +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/PersonNameComparator.java @@ -0,0 +1,44 @@ +package fr.dossierfacile.process.file.util; + +import org.apache.commons.lang3.StringUtils; + +import java.text.Normalizer; +import java.util.Arrays; +import java.util.List; + +public class PersonNameComparator { + private static String normalizeName(String name) { + if (name == null) + return null; + String normalized = Normalizer.normalize(name, Normalizer.Form.NFD); + return normalized.replace('-', ' ') + .replaceAll("[\\p{InCombiningDiacriticalMarks}]", "").toUpperCase().trim(); + } + + /** + * fullname start by LastName + **/ + public static boolean bearlyEqualsTo(String fullName, String lastName, String firstName) { + if (StringUtils.isEmpty(fullName) || StringUtils.isEmpty(firstName) || StringUtils.isEmpty(lastName)) + return false; + String full = normalizeName(fullName); + String[] allFull = full.split(" "); + if (allFull.length < 2) + return false; + // Lastname can be all the string excluding the LAST one; + List lastnames = Arrays.stream(allFull).limit(allFull.length - 1).toList(); + // firstname can be all the string excluding the FIRST one; + List firstnames = Arrays.stream(allFull).skip(1).toList(); + + // + String givenLastName = normalizeName(lastName); + List givenLastNames = Arrays.stream(givenLastName.split(" ")).toList(); + String givenFirstName = normalizeName(firstName); + List givenFirstNames = Arrays.stream(givenFirstName.split(" ")).toList(); + + // if there is one matching on firstname and one matching on lastname the user should be the same + return lastnames.stream().anyMatch(givenLastNames::contains) + && firstnames.stream().anyMatch(givenFirstNames::contains); + } + +} diff --git a/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationServiceTest.java b/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationServiceTest.java index 2b33e047f..344601bf3 100644 --- a/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationServiceTest.java +++ b/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationServiceTest.java @@ -22,6 +22,7 @@ private Document buildValidTaxDocument() throws Exception { .lastName("DUPONT") .build(); BarCodeFileAnalysis barCodeFileAnalysis = BarCodeFileAnalysis.builder() + .documentType(BarCodeDocumentType.TAX_ASSESSMENT) .verifiedData( new ObjectMapper().readValue(""" { @@ -61,34 +62,18 @@ public void document_full_test() throws Exception { Assertions.assertThat(report.getAnalysisStatus()).isEqualTo(DocumentAnalysisStatus.CHECKED); } -/* @Test - public void compare_name_should_be_ok() { - String givenLastName = "Jean Phillipe"; - String givenFirstName = "DE LA MARCHE"; - String fullName = "DE LA MARCHE PICQUET Edouard Jean Phillipe"; - } - - @Test - public void compare_name_should_be_ko() { - String givenLastName = "Jean Phillipe"; - String givenFirstName = "DE LA MARCHE"; - String fullName = "PICQUET Edouard Jean Phillipe"; - } - - @Test - public void compare_name_should_be_ok2() { - String givenLastName = "Jean Phillipe"; - String givenFirstName = "MARCHE"; - String fullName = "ROSE-MARCHE Jean"; - } + public void document_full_test_wrong_firstname() throws Exception { + Document document = buildValidTaxDocument(); + document.getTenant().setFirstName("Joseph"); + DocumentAnalysisReport report = DocumentAnalysisReport.builder() + .document(document) + .brokenRules(new LinkedList<>()) + .build(); + incomeTaxRulesValidationService.process(document, report); - @Test - public void compare_name_should_be_ok_basc() { - String givenLastName = "Jean"; - String givenFirstName = "MARCHE"; - String fullName = "MARCHE Jean"; + Assertions.assertThat(report.getAnalysisStatus()).isEqualTo(DocumentAnalysisStatus.UNDEFINED); + Assertions.assertThat(report.getBrokenRules()).hasSize(1); + Assertions.assertThat(report.getBrokenRules().get(0)).matches(docRule -> docRule.getRule() == DocumentRule.R_TAX_NAMES); } - - */ } \ No newline at end of file diff --git a/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/util/PersonNameComparatorTest.java b/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/util/PersonNameComparatorTest.java new file mode 100644 index 000000000..9fd466bc0 --- /dev/null +++ b/dossierfacile-process-file/src/test/java/fr/dossierfacile/process/file/util/PersonNameComparatorTest.java @@ -0,0 +1,46 @@ +package fr.dossierfacile.process.file.util; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class PersonNameComparatorTest { + @Test + public void compare_name_should_be_ok() { + String givenFirstName = "Jean Phillipe"; + String givenLastName = "DE LA MARCHE"; + String fullName = "DE LA MARCHE PICQUET Edouard Jean Phillipe"; + Assertions.assertEquals(true, PersonNameComparator.bearlyEqualsTo(fullName, givenLastName, givenFirstName)); + } + + @Test + public void compare_name_should_be_ko() { + String givenFirstName = "Jean Phillipe"; + String givenLastName = "DE LA MARCHE"; + String fullName = "PICQUET Edouard Jean Phillipe"; + Assertions.assertEquals(false, PersonNameComparator.bearlyEqualsTo(fullName, givenLastName, givenFirstName)); + } + + @Test + public void compare_name_should_be_ok2() { + String givenFirstName = "Jean Phillipe"; + String givenLastName = "MARCHE"; + String fullName = "ROSE-MARCHE Jean"; + Assertions.assertEquals(true, PersonNameComparator.bearlyEqualsTo(fullName, givenLastName, givenFirstName)); + } + + @Test + public void compare_name_should_be_ok_basc() { + String givenFirstName = "Jean"; + String givenLastName = "MARCHE"; + String fullName = "MARCHE Jean"; + Assertions.assertEquals(true, PersonNameComparator.bearlyEqualsTo(fullName, givenLastName, givenFirstName)); + } + + @Test + public void compare_name_should_be_nok_reverse() { + String givenFirstName = "Marche"; + String givenLastName = "Jean"; + String fullName = "MARCHE Jean"; + Assertions.assertEquals(false, PersonNameComparator.bearlyEqualsTo(fullName, givenLastName, givenFirstName)); + } +} \ No newline at end of file From b7814bdd78a118f475cf86e7e7d1d7db0b83bd8f Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 3 Jan 2024 12:54:41 +0100 Subject: [PATCH 17/19] feat: improve tax income parsing and rules --- .../IncomeTaxRulesValidationService.java | 2 +- .../file/service/ocr/TaxIncomeParser.java | 24 ++++++++++++------- .../file/util/PersonNameComparator.java | 6 +++-- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java index c5ea22bb5..a5a7cc639 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java @@ -89,7 +89,7 @@ public DocumentAnalysisReport process(Document document, @NotNull DocumentAnalys && parsedDocument.getDeclarant1Nom() != null && parsedDocument.getRevenuFiscalDeReference() != null && (!qrDocument.getAnneeDesRevenus().equals(parsedDocument.getAnneeDesRevenus()) - || !qrDocument.getDeclarant1Nom().equalsIgnoreCase(parsedDocument.getDeclarant1Nom()) + || !PersonNameComparator.equalsWithNormalization(qrDocument.getDeclarant1Nom(), parsedDocument.getDeclarant1Nom()) || !qrDocument.getRevenuFiscalDeReference().equals(parsedDocument.getRevenuFiscalDeReference()) )) { log.error("Le 2DDoc code ne correspond pas au contenu du document tenantId:" + document.getTenant().getId()); diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/ocr/TaxIncomeParser.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/ocr/TaxIncomeParser.java index b6ea288a4..56cb0b43d 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/ocr/TaxIncomeParser.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/ocr/TaxIncomeParser.java @@ -30,7 +30,7 @@ public class TaxIncomeParser implements OcrParser { static final TaxIncomeZones TEMPLATE_2023 = new TaxIncomeZones(new Rectangle(120, 10, 400, 65), new Rectangle(17, 176, 200, 180), new Rectangle(250, 170, 300, 100), new Rectangle(226, 595, 325, 55)); static final TaxIncomeZones TEMPLATE_2020 = new TaxIncomeZones(new Rectangle(140, 40, 325, 65), new Rectangle(40, 155, 200, 180), new Rectangle(270, 130, 300, 100), new Rectangle(255, 570, 315, 55)); private final Pattern incomeYearPattern = Pattern.compile("revenus de (\\d{4})"); - private final Pattern incomeAmountPattern = Pattern.compile("Revenu fiscal de référence : ([\\d \\s]+)"); + private final Pattern incomeAmountPattern = Pattern.compile("Revenu fiscal de référence : ([\\dOlI|S \\s]+)"); private final Pattern partCountPattern = Pattern.compile("Nombre de parts : ([\\d\\s,]+)"); private final Pattern declarant2NameInAddressPattern = Pattern.compile("OU (.+)\n"); private final Pattern declarant1Pattern = Pattern.compile("Déclarant 1 \\(C\\): ([\\s\\w]+)|Numéro fiscal \\(C\\) : ([\\s\\w]+)|"); @@ -38,6 +38,12 @@ public class TaxIncomeParser implements OcrParser { private final TaxIncomeLeafParser taxIncomeLeafParser; private transient volatile Tesseract tesseract; + String fixNumber(String str){ + return str.replaceAll("O", "0").replaceAll("[lI|]", "1").replaceAll("S", "5").replaceAll("\\s", ""); + } + String fixLetter(String str){ + return str.replaceAll("0", "O").replaceAll("5", "S").trim(); + } void init() { if (tesseract == null) { this.tesseract = new Tesseract(); @@ -84,29 +90,29 @@ public TaxIncomeMainFile parse(File file) { String zoneTitle = tesseract.doOCR(image, TaxIncomeZones.scale(zones.zoneTitle, scale)); Matcher matcher = incomeYearPattern.matcher(zoneTitle); if (matcher.find()) { - result.setAnneeDesRevenus(Integer.parseInt(matcher.group(1).replaceAll("\\s", ""))); + result.setAnneeDesRevenus(Integer.parseInt(fixNumber(matcher.group(1)))); } else { log.warn("Income year not found"); - continue; + continue; // not need to continue the parsing is failed } String zoneRef = tesseract.doOCR(image, TaxIncomeZones.scale(zones.zoneRef, scale)); Matcher declarant1Matcher = declarant1Pattern.matcher(zoneRef); if (declarant1Matcher.find()) { - result.setDeclarant2Nom(declarant1Matcher.group(1)); + result.setDeclarant1NumFiscal(declarant1Matcher.group(1)); } else { - log.info("\"Déclarant 1\" in address not found \n"); + log.info("\"Déclarant 1 fiscal number\" not found \n"); } String zoneAddress = tesseract.doOCR(image, TaxIncomeZones.scale(zones.zoneAddress, scale)); String declarant1Name = zoneAddress.substring(0, zoneAddress.indexOf('\n')); if (declarant1Name != null) { - result.setDeclarant1Nom(declarant1Name); + result.setDeclarant1Nom(fixLetter(declarant1Name)); } else { - log.warn("\"Déclarant 1\" in adress not found \n");//GDPR + log.warn("\"Déclarant 1\" in address not found \n");//GDPR continue; } Matcher declarant2NameMatcher = declarant2NameInAddressPattern.matcher(zoneAddress); if (declarant2NameMatcher.find()) { - result.setDeclarant2Nom(declarant2NameMatcher.group(1)); + result.setDeclarant2Nom(fixLetter(declarant2NameMatcher.group(1))); } else { log.info("\"Déclarant 2\" in address not found \n");//GDPR } @@ -121,7 +127,7 @@ public TaxIncomeMainFile parse(File file) { } Matcher incomeAmountMatcher = incomeAmountPattern.matcher(zoneRevenuPart); if (incomeAmountMatcher.find()) { - result.setRevenuFiscalDeReference(Integer.parseInt(incomeAmountMatcher.group(1).replaceAll("\\s", ""))); + result.setRevenuFiscalDeReference(Integer.parseInt(fixNumber(incomeAmountMatcher.group(1)))); break; } else { log.warn("\"Revenu fiscal de référence\" not found" + zoneRevenuPart); diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/PersonNameComparator.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/PersonNameComparator.java index 23db4232d..8eec0c9a2 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/PersonNameComparator.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/util/PersonNameComparator.java @@ -11,10 +11,12 @@ private static String normalizeName(String name) { if (name == null) return null; String normalized = Normalizer.normalize(name, Normalizer.Form.NFD); - return normalized.replace('-', ' ') + return normalized.replace('-', ' ').replace('.', ' ') .replaceAll("[\\p{InCombiningDiacriticalMarks}]", "").toUpperCase().trim(); } - + public static boolean equalsWithNormalization(String fullName, String fullNameToCompare) { + return normalizeName(fullName).equals(normalizeName(fullNameToCompare)); + } /** * fullname start by LastName **/ From 75221b06311344870e65a3743889378607aee222 Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 3 Jan 2024 13:31:25 +0100 Subject: [PATCH 18/19] feat: set check on the name critical --- .../main/java/fr/dossierfacile/common/entity/DocumentRule.java | 2 +- .../service/documentrules/IncomeTaxRulesValidationService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/DocumentRule.java b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/DocumentRule.java index fddb3b30c..e103ab9c0 100644 --- a/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/DocumentRule.java +++ b/dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/DocumentRule.java @@ -13,7 +13,7 @@ public enum DocumentRule { R_TAX_LEAF( Level.CRITICAL, "Veuillez fournir les feuillets des avis"), R_TAX_ALL_LEAF( Level.WARN, "Veuillez fournir tous les feuillets des avis"),// feuillet 1 founi R_TAX_N3( Level.CRITICAL, "Les avis d'imposition antérieur à N-3 ne sont pas autorisé"), - R_TAX_NAMES( Level.WARN, "Les noms et prénoms ne correspondent pas"); + R_TAX_NAMES( Level.CRITICAL, "Les noms et prénoms ne correspondent pas"); public enum Level{ CRITICAL, WARN diff --git a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java index a5a7cc639..abca65e3e 100644 --- a/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java +++ b/dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/IncomeTaxRulesValidationService.java @@ -154,7 +154,7 @@ public DocumentAnalysisReport process(Document document, @NotNull DocumentAnalys if (!(PersonNameComparator.bearlyEqualsTo(qrDocument.getDeclarant1Nom(), lastName, firstName) || (qrDocument.getDeclarant2Nom() != null && - PersonNameComparator.bearlyEqualsTo(qrDocument.getDeclarant1Nom(), lastName, firstName)) + PersonNameComparator.bearlyEqualsTo(qrDocument.getDeclarant2Nom(), lastName, firstName)) )) { log.error("Le nom/prenom ne correpond pas à l'uilitsation tenantId:" + document.getTenant().getId() + " firstname: " + firstName); brokenRules.add(DocumentBrokenRule.builder() From 1c016cbcadb8b56a3edc5fa68d899eb89f11a776 Mon Sep 17 00:00:00 2001 From: Fabien Date: Wed, 3 Jan 2024 14:34:59 +0100 Subject: [PATCH 19/19] fix: date comparison with null --- .../amqp/WatermarkDFDocumentConsumer.java | 4 ++-- .../pdfgenerator/service/DocumentServiceImpl.java | 12 ++++++------ .../service/interfaces/DocumentService.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/amqp/WatermarkDFDocumentConsumer.java b/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/amqp/WatermarkDFDocumentConsumer.java index ccde45e39..f802e4783 100644 --- a/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/amqp/WatermarkDFDocumentConsumer.java +++ b/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/amqp/WatermarkDFDocumentConsumer.java @@ -38,9 +38,9 @@ public void receiveMessage(Message message) throws Exception { Map data = gson.fromJson(new String(message.getBody()), Map.class); Long documentId = Long.valueOf(data.get("id")); if (documentService.documentIsUpToDateAt(msgTimestamp, documentId)) { - LocalDateTime executionDateTime = LocalDateTime.now(); + Long executionTimestamp = System.currentTimeMillis(); StorageFile watermarkFile = pdfGeneratorService.generateBOPdfDocument(documentService.getDocument(documentId)); - documentService.saveWatermarkFileAt(executionDateTime, watermarkFile, documentId); + documentService.saveWatermarkFileAt(executionTimestamp, watermarkFile, documentId); } else { log.debug("Ignore document pdf generation cause document is NOT up to date"); } diff --git a/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/DocumentServiceImpl.java b/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/DocumentServiceImpl.java index ea349b5b7..96de4337b 100644 --- a/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/DocumentServiceImpl.java +++ b/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/DocumentServiceImpl.java @@ -27,7 +27,7 @@ public boolean documentIsUpToDateAt(Long timestamp, Long documentId) { LocalDateTime dateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault()); Document document = documentRepository.findById(documentId).orElse(null); - return document != null && ( document.getLastModifiedDate() == null || dateTime.isAfter(document.getLastModifiedDate())); + return document != null && (document.getLastModifiedDate() == null || dateTime.isAfter(document.getLastModifiedDate())); } @Override @@ -37,15 +37,15 @@ public Document getDocument(Long documentId) { @Transactional @Override - public void saveWatermarkFileAt(LocalDateTime executionDateTime, StorageFile watermarkFile, Long documentId) { + public void saveWatermarkFileAt(Long executionTimestamp, StorageFile watermarkFile, Long documentId) { Document document = documentRepository.findById(documentId).orElse(null); - if (document == null || executionDateTime.isBefore(document.getLastModifiedDate())) { - fileStorageService.delete(watermarkFile); - log.warn("PDF Generation execution is deprecated for documentId=" + documentId); - } else { + if (documentIsUpToDateAt(executionTimestamp, document.getId())) { document.setWatermarkFile(watermarkFile); documentRepository.save(document); log.warn("PDF Generation execution is a success for documentId=" + documentId); + } else { + fileStorageService.delete(watermarkFile); + log.warn("PDF Generation execution is deprecated for documentId=" + documentId); } } } diff --git a/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/interfaces/DocumentService.java b/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/interfaces/DocumentService.java index 880d39ac8..4f8127ff5 100644 --- a/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/interfaces/DocumentService.java +++ b/dossierfacile-pdf-generator/src/main/java/fr/dossierfacile/api/pdfgenerator/service/interfaces/DocumentService.java @@ -8,7 +8,7 @@ public interface DocumentService { boolean documentIsUpToDateAt(Long timestamp, Long documentId); - void saveWatermarkFileAt(LocalDateTime executionDateTime, StorageFile watermarkFile, Long documentId); + void saveWatermarkFileAt(Long executionTimestamp, StorageFile watermarkFile, Long documentId); Document getDocument(Long documentId); }