From 09410143d3bbde7b9c34a6576df8e1c0d7e8606a Mon Sep 17 00:00:00 2001 From: Anagh Hegde Date: Tue, 9 Jul 2024 17:15:38 +0530 Subject: [PATCH] chore: refactor customJs lib import flow for pg transaction support (#34622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Please refer this document for more details - https://www.notion.so/appsmith/Transaction-Handling-in-PG-468cf8d4255749c3915699e59e91dc2f ## Automation /ok-to-test tags="@tag.Git, @tag.ImportExport, @tag.Templates" ### :mag: Cypress test results > [!TIP] > 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉 > Workflow run: > Commit: b9dfe1d758410e831f751f1fe1047373c1137513 > Cypress dashboard. > Tags: `@tag.Git, @tag.ImportExport, @tag.Templates` > Spec: >
Tue, 09 Jul 2024 11:25:30 UTC ## Communication Should the DevRel and Marketing teams inform users about this change? - [ ] Yes - [ ] No ## Summary by CodeRabbit - **New Features** - Introduced support for managing custom JavaScript libraries, including dry run operations. - **Enhancements** - Enhanced dry operation handling with new methods and improved logic for managing custom JavaScript libraries. --- .../ce/MappedImportableResourcesCE_DTO.java | 3 ++ .../jslibs/base/CustomJSLibServiceCE.java | 7 ++++ .../jslibs/base/CustomJSLibServiceCEImpl.java | 36 ++++++++++++++++++- .../repositories/DryOperationRepository.java | 20 ++++++++++- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedImportableResourcesCE_DTO.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedImportableResourcesCE_DTO.java index 58363cdf2ef..2424ec26dd6 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedImportableResourcesCE_DTO.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/dtos/ce/MappedImportableResourcesCE_DTO.java @@ -3,6 +3,7 @@ import com.appsmith.external.models.Datasource; import com.appsmith.external.models.DatasourceStorage; import com.appsmith.server.domains.Context; +import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.dtos.CustomJSLibContextDTO; import com.appsmith.server.dtos.ImportActionCollectionResultDTO; import com.appsmith.server.dtos.ImportActionResultDTO; @@ -50,4 +51,6 @@ public class MappedImportableResourcesCE_DTO { Map> datasourceDryRunQueries = new HashMap<>(); Map> datasourceStorageDryRunQueries = new HashMap<>(); + + Map> customJSLibsDryOps = new HashMap<>(); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/base/CustomJSLibServiceCE.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/base/CustomJSLibServiceCE.java index ea9731a8608..bf588a0fbf6 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/base/CustomJSLibServiceCE.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/base/CustomJSLibServiceCE.java @@ -9,6 +9,7 @@ import reactor.core.publisher.Mono; import java.util.List; +import java.util.Map; import java.util.Set; public interface CustomJSLibServiceCE extends CrudService { @@ -32,6 +33,12 @@ Mono> getAllJSLibsInContext( Mono persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO( CustomJSLib jsLib, Boolean isForceInstall); + Mono persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO( + CustomJSLib jsLib, + Boolean isForceInstall, + Map> customJSLibsDryOps, + boolean isDryOps); + Flux getAllVisibleJSLibsInContext( @NotNull String contextId, CreatorContextType contextType, String branchName, Boolean isViewMode); } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/base/CustomJSLibServiceCEImpl.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/base/CustomJSLibServiceCEImpl.java index 90e276c5db5..7abf231ae98 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/base/CustomJSLibServiceCEImpl.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/jslibs/base/CustomJSLibServiceCEImpl.java @@ -4,6 +4,7 @@ import com.appsmith.server.domains.Application; import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.dtos.CustomJSLibContextDTO; +import com.appsmith.server.dtos.DBOpsType; import com.appsmith.server.jslibs.context.ContextBasedJsLibService; import com.appsmith.server.repositories.CustomJSLibRepository; import com.appsmith.server.services.AnalyticsService; @@ -14,8 +15,10 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -73,11 +76,27 @@ public Mono addJSLibsToContext( @Override public Mono persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO( CustomJSLib jsLib, Boolean isForceInstall) { + return persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO(jsLib, isForceInstall, null, false); + } + + @Override + public Mono persistCustomJSLibMetaDataIfDoesNotExistAndGetDTO( + CustomJSLib jsLib, + Boolean isForceInstall, + Map> customJSLibsDryOps, + boolean isDryOps) { return repository .findUniqueCustomJsLib(jsLib) // Read more why Mono.defer is used here. // https://stackoverflow.com/questions/54373920/mono-switchifempty-is-always-called - .switchIfEmpty(Mono.defer(() -> repository.save(jsLib))) + .switchIfEmpty(Mono.defer(() -> { + if (isDryOps) { + jsLib.updateForBulkWriteOperation(); + addDryOpsForEntity(DBOpsType.SAVE.name(), customJSLibsDryOps, jsLib); + return Mono.just(jsLib); + } + return repository.save(jsLib); + })) .flatMap(foundJSLib -> { /* The first check is to make sure that we are able to detect any previously truncated data and overwrite it the next time we receive valid data. @@ -86,6 +105,10 @@ public Mono persistCustomJSLibMetaDataIfDoesNotExistAndGe */ if ((jsLib.getDefs().length() > foundJSLib.getDefs().length()) || isForceInstall) { jsLib.setId(foundJSLib.getId()); + if (isDryOps) { + addDryOpsForEntity(DBOpsType.SAVE.name(), customJSLibsDryOps, jsLib); + return Mono.just(jsLib); + } return repository.save(jsLib); } @@ -141,4 +164,15 @@ public Flux getAllVisibleJSLibsInContext( .getAllVisibleJSLibContextDTOFromContext(contextId, branchName, isViewMode) .flatMapMany(repository::findCustomJsLibsInContext); } + + private void addDryOpsForEntity( + String queryType, Map> dryRunOpsMap, CustomJSLib createdCustomJsLib) { + if (dryRunOpsMap.containsKey(queryType)) { + dryRunOpsMap.get(queryType).add(createdCustomJsLib); + } else { + List customJsLibList = new ArrayList<>(); + customJsLibList.add(createdCustomJsLib); + dryRunOpsMap.put(queryType, customJsLibList); + } + } } diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/DryOperationRepository.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/DryOperationRepository.java index d957efc073f..071ee83fbf4 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/DryOperationRepository.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/repositories/DryOperationRepository.java @@ -2,6 +2,7 @@ import com.appsmith.external.models.Datasource; import com.appsmith.external.models.DatasourceStorage; +import com.appsmith.server.domains.CustomJSLib; import com.appsmith.server.dtos.MappedImportableResourcesDTO; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; @@ -22,6 +23,8 @@ public class DryOperationRepository { private final DatasourceStorageRepository datasourceStorageRepository; + private final CustomJSLibRepository customJSLibRepository; + private Map, AppsmithRepository> repoByEntityClass; @PostConstruct @@ -29,6 +32,7 @@ public void init() { final Map, AppsmithRepository> map = new HashMap<>(); map.put(Datasource.class, datasourceRepository); map.put(DatasourceStorage.class, datasourceStorageRepository); + map.put(CustomJSLib.class, customJSLibRepository); repoByEntityClass = Collections.unmodifiableMap(map); } @@ -44,6 +48,10 @@ public Flux saveDatasourceStorageToDb(List return datasourceStorageRepository.saveAll(datasourceStorage); } + private Flux saveCustomJSLibToDb(List customJSLibs) { + return customJSLibRepository.saveAll(customJSLibs); + } + public Mono executeAllDbOps(MappedImportableResourcesDTO mappedImportableResourcesDTO) { Flux> datasourceFLux = Flux.fromIterable(mappedImportableResourcesDTO @@ -65,6 +73,16 @@ public Mono executeAllDbOps(MappedImportableResourcesDTO mappedImportableR .get(key); return saveDatasourceStorageToDb(datasourceStorageList).collectList(); }); - return Flux.merge(datasourceFLux, datasourceStorageFLux).then(); + + Flux> customJSLibFLux = Flux.fromIterable( + mappedImportableResourcesDTO.getCustomJSLibsDryOps().keySet()) + .flatMap(key -> { + List customJSLibList = + mappedImportableResourcesDTO.getCustomJSLibsDryOps().get(key); + return saveCustomJSLibToDb(customJSLibList).collectList(); + }); + + return Flux.merge(datasourceFLux, datasourceStorageFLux, customJSLibFLux) + .then(); } }