Reduce memory consumption (#40)

* Use sequence instead of list to utilize lazy transformation. * Implement dynamic batch size for commit documents (taking into account number of files).
octopusden · May 24, 2024 · 065ab1f · 065ab1f
1 parent f20cd5b
commit 065ab1f
Show file tree

Hide file tree

Showing 14 changed files with 256 additions and 240 deletions.
diff --git a/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/GiteaIndexerController.kt b/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/GiteaIndexerController.kt
@@ -66,7 +66,7 @@ class GiteaIndexerController(
         if (eventType == "create" && event == "create") {
             with(objectMapper.readValue(payload, GiteaCreateRefEvent::class.java)) {
                 log.info(
-                    "Register `{}` {} creation in `{}` {} repository",
+                    "Register '{}' {} creation in {} {} repository",
                     ref,
                     refType.jsonValue,
                     repository.fullName,
@@ -77,7 +77,7 @@ class GiteaIndexerController(
         } else if (eventType == "delete" && event == "delete") {
             with(objectMapper.readValue(payload, GiteaDeleteRefEvent::class.java)) {
                 log.info(
-                    "Register `{}` {} deletion in `{}` {} repository",
+                    "Register '{}' {} deletion in {} {} repository",
                     ref,
                     refType.jsonValue,
                     repository.fullName,
@@ -87,12 +87,12 @@ class GiteaIndexerController(
             }
         } else if (eventType == "push" && event == "push") {
             with(objectMapper.readValue(payload, GiteaPushEvent::class.java)) {
-                log.info("Register {} commit(s) in `{}` {} repository", commits.size, repository.fullName, GITEA)
+                log.info("Register {} commit(s) in {} {} repository", commits.size, repository.fullName, GITEA)
                 giteaIndexerService.registerGiteaPushEvent(this)
             }
         } else if (eventType == "pull_request" && event == "pull_request") {
             with(objectMapper.readValue(payload, GiteaPullRequestEvent::class.java)) {
-                log.info("Register {} pull request in `{}` {} repository", action, repository.fullName, GITEA)
+                log.info("Register {} pull request in {} {} repository", action, repository.fullName, GITEA)
                 giteaIndexerService.registerGiteaPullRequestEvent(this)
             }
         }

diff --git a/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/RepositoryController.kt b/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/RepositoryController.kt
@@ -51,7 +51,7 @@ class RepositoryController(
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
         log.info(
-            "Get commits ({},{}] in `{}` repository",
+            "Get commits ({},{}] in {} repository",
             (fromHashOrRef ?: fromDate?.toString()).orEmpty(),
             toHashOrRef,
             sshUrl
@@ -69,7 +69,7 @@ class RepositoryController(
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
         log.info(
-            "Get commits ({},{}] with files (limit {}) in `{}` repository",
+            "Get commits ({},{}] with files (limit {}) in {} repository",
             (fromHashOrRef ?: fromDate?.toString()).orEmpty(),
             toHashOrRef,
             commitFilesLimit,
@@ -87,7 +87,7 @@ class RepositoryController(
         @RequestParam("hashOrRef") hashOrRef: String,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Get commit {} in `{}` repository", hashOrRef, sshUrl)
+        log.info("Get commit {} in {} repository", hashOrRef, sshUrl)
         vcsManager.getCommit(sshUrl, hashOrRef)
     }
 
@@ -98,7 +98,7 @@ class RepositoryController(
         @RequestParam("commitFilesLimit", defaultValue = "0") commitFilesLimit: Int,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Get commit {} with files (limit {}) in `{}` repository", hashOrRef, commitFilesLimit, sshUrl)
+        log.info("Get commit {} with files (limit {}) in {} repository", hashOrRef, commitFilesLimit, sshUrl)
         vcsManager.getCommitWithFiles(sshUrl, hashOrRef).applyCommitFilesLimit(commitFilesLimit)
     }
 
@@ -111,7 +111,7 @@ class RepositoryController(
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
         log.info(
-            "Find issue keys in commits ({},{}] in `{}` repository",
+            "Find issue keys in commits ({},{}] in {} repository",
             (fromHashOrRef ?: fromDate?.toString()).orEmpty(),
             toHashOrRef,
             sshUrl
@@ -128,7 +128,7 @@ class RepositoryController(
         @RequestParam("sshUrl") sshUrl: String,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Get tags in `{}` repository", sshUrl)
+        log.info("Get tags in {} repository", sshUrl)
         RepositoryResponse(vcsManager.getTags(sshUrl))
     }.data.sorted()
 
@@ -151,7 +151,7 @@ class RepositoryController(
         @RequestBody createPullRequest: CreatePullRequest
     ): PullRequest {
         log.info(
-            "Create pull request ({} -> {}) in `{}` repository",
+            "Create pull request ({} -> {}) in {} repository",
             sshUrl,
             createPullRequest.sourceBranch,
             createPullRequest.targetBranch

diff --git a/...er/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/RepositoryControllerOld.kt b/...er/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/RepositoryControllerOld.kt
@@ -58,9 +58,8 @@ class RepositoryControllerOld(
         @RequestParam("fromDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) fromDate: Date?,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processJob(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Get commits ({},{}] in `{}` repository", (from ?: fromDate?.toString()).orEmpty(), to, vcsPath)
-        val commits = vcsManager.getCommits(vcsPath, from, fromDate, to).map { it.toOld() }
-        RepositoryResponse(commits)
+        log.info("Get commits ({},{}] in {} repository", (from ?: fromDate?.toString()).orEmpty(), to, vcsPath)
+        RepositoryResponse(vcsManager.getCommits(vcsPath, from, fromDate, to).map { it.toOld() })
     }.data
 
     @Deprecated(
@@ -84,7 +83,7 @@ class RepositoryControllerOld(
         @RequestParam("vcsPath") vcsPath: String,
         @RequestParam("commitId") commitIdOrRef: String
     ): CommitOld {
-        log.info("Get commit {} in `{}` repository", commitIdOrRef, vcsPath)
+        log.info("Get commit {} in {} repository", commitIdOrRef, vcsPath)
         return vcsManager.getCommit(vcsPath, commitIdOrRef).toOld()
     }
 
@@ -97,14 +96,13 @@ class RepositoryControllerOld(
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processJob(requestId ?: UUID.randomUUID().toString()) {
         log.info(
-            "Find issue keys in commits ({},{}] in `{}` repository",
+            "Find issue keys in commits ({},{}] in {} repository",
             (from ?: fromDate?.toString()).orEmpty(), to, vcsPath
         )
-        val issues = vcsManager.getCommits(vcsPath, from, fromDate, to)
-            .map { it.toOld() }
-            .flatMap { IssueKeyParser.findIssueKeys(it.message) }
-            .distinct()
-        RepositoryResponse(issues)
+        RepositoryResponse(
+            vcsManager.getCommits(vcsPath, from, fromDate, to)
+                .flatMap { IssueKeyParser.findIssueKeys(it.message) }.distinct()
+        )
     }.data
 
     @GetMapping("issues/{issueKey}", produces = [MediaType.APPLICATION_JSON_VALUE])
@@ -113,8 +111,7 @@ class RepositoryControllerOld(
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processJob(requestId ?: UUID.randomUUID().toString()) {
         log.info("Find commits by issue key {}", issueKey)
-        val commits = vcsManager.findCommits(issueKey).map { it.toOld() }
-        RepositoryResponse(commits)
+        RepositoryResponse(vcsManager.findCommits(issueKey).map { it.toOld() })
     }.data
 
 
@@ -123,7 +120,7 @@ class RepositoryControllerOld(
         @RequestParam("vcsPath") vcsPath: String,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processJob(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Get tags in `{}` repository", vcsPath)
+        log.info("Get tags in {} repository", vcsPath)
         RepositoryResponse(vcsManager.getTags(vcsPath).map { it.toOld() })
     }.data
 
@@ -147,7 +144,7 @@ class RepositoryControllerOld(
         @RequestBody createPullRequest: CreatePullRequest
     ): PullRequestResponse {
         log.info(
-            "Create pull request ({} -> {}) in `{}` repository",
+            "Create pull request ({} -> {}) in {} repository",
             vcsPath,
             createPullRequest.sourceBranch,
             createPullRequest.targetBranch

diff --git a/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/RepositoryControllerV1.kt b/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/RepositoryControllerV1.kt
@@ -58,13 +58,13 @@ class RepositoryControllerV1(
         @RequestParam("fromDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) fromDate: Date?,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Get commits ({},{}] in `{}` repository", (from ?: fromDate?.toString()).orEmpty(), to, sshUrl)
+        log.info("Get commits ({},{}] in {} repository", (from ?: fromDate?.toString()).orEmpty(), to, sshUrl)
         RepositoryResponse(vcsManager.getCommits(sshUrl, from, fromDate, to))
     }.data.sorted().map { it.toV1() }
 
     @GetMapping("commit", produces = [MediaType.APPLICATION_JSON_VALUE])
     fun getCommit(@RequestParam("sshUrl") sshUrl: String, @RequestParam("commitId") commitId: String): CommitV1 {
-        log.info("Get commit {} in `{}` repository", commitId, sshUrl)
+        log.info("Get commit {} in {} repository", commitId, sshUrl)
         return vcsManager.getCommit(sshUrl, commitId).toV1()
     }
 
@@ -77,7 +77,7 @@ class RepositoryControllerV1(
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
         log.info(
-            "Find issue keys in commits ({},{}] in `{}` repository",
+            "Find issue keys in commits ({},{}] in {} repository",
             (from ?: fromDate?.toString()).orEmpty(), to, sshUrl
         )
         RepositoryResponse(
@@ -92,7 +92,7 @@ class RepositoryControllerV1(
         @RequestParam("sshUrl") sshUrl: String,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Get tags in `{}` repository", sshUrl)
+        log.info("Get tags in {} repository", sshUrl)
         RepositoryResponse(vcsManager.getTags(sshUrl))
     }.data.sorted().map { it.toV1() }
 
@@ -114,7 +114,7 @@ class RepositoryControllerV1(
         @RequestParam("sshUrl") sshUrl: String, @RequestBody createPullRequest: CreatePullRequest
     ): PullRequest {
         log.info(
-            "Create pull request ({} -> {}) in `{}` repository",
+            "Create pull request ({} -> {}) in {} repository",
             sshUrl,
             createPullRequest.sourceBranch,
             createPullRequest.targetBranch

diff --git a/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/dto/RepositoryResponse.kt b/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/dto/RepositoryResponse.kt
@@ -2,4 +2,4 @@ package org.octopusden.octopus.vcsfacade.dto
 
 import org.octopusden.octopus.vcsfacade.client.common.dto.VcsFacadeResponse
 
-data class RepositoryResponse<T>(val data: List<T>) : VcsFacadeResponse
+data class RepositoryResponse<T>(val data: Sequence<T>) : VcsFacadeResponse
diff --git a/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/OpenSearchService.kt b/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/OpenSearchService.kt
@@ -26,23 +26,23 @@ import org.octopusden.octopus.vcsfacade.dto.VcsServiceType
 interface OpenSearchService {
     fun findRepositoriesInfoByRepositoryType(type: VcsServiceType): Set<RepositoryInfoDocument>
     fun findRepositoryInfoById(repositoryId: String): RepositoryInfoDocument?
-    fun saveRepositoriesInfo(repositoriesInfo: List<RepositoryInfoDocument>)
+    fun saveRepositoriesInfo(repositoriesInfo: Sequence<RepositoryInfoDocument>)
     fun deleteRepositoryInfoById(repositoryId: String)
-    fun findRefsByRepositoryId(repositoryId: String): Set<RefDocument>
-    fun saveRefs(refs: List<RefDocument>)
-    fun deleteRefsByIds(refsIds: List<String>)
+    fun findRefsIdsByRepositoryId(repositoryId: String): Set<String>
+    fun saveRefs(refs: Sequence<RefDocument>)
+    fun deleteRefsByIds(refsIds: Sequence<String>)
     fun deleteRefsByRepositoryId(repositoryId: String)
-    fun findCommitsByRepositoryId(repositoryId: String): Set<CommitDocument>
-    fun saveCommits(commits: List<CommitDocument>)
-    fun deleteCommitsByIds(commitsIds: List<String>)
+    fun findCommitsIdsByRepositoryId(repositoryId: String): Set<String>
+    fun saveCommits(commits: Sequence<CommitDocument>)
+    fun deleteCommitsByIds(commitsIds: Sequence<String>)
     fun deleteCommitsByRepositoryId(repositoryId: String)
-    fun findPullRequestsByRepositoryId(repositoryId: String): Set<PullRequestDocument>
-    fun savePullRequests(pullRequests: List<PullRequestDocument>)
-    fun deletePullRequestsByIds(pullRequestsIds: List<String>)
+    fun findPullRequestsIdsByRepositoryId(repositoryId: String): Set<String>
+    fun savePullRequests(pullRequests: Sequence<PullRequestDocument>)
+    fun deletePullRequestsByIds(pullRequestsIds: Sequence<String>)
     fun deletePullRequestsByRepositoryId(repositoryId: String)
-    fun findBranchesByIssueKey(issueKey: String): List<RefDocument>
-    fun findCommitsByIssueKey(issueKey: String): List<CommitDocument>
-    fun findPullRequestsByIssueKey(issueKey: String): List<PullRequestDocument>
+    fun findBranchesByIssueKey(issueKey: String): Sequence<RefDocument>
+    fun findCommitsByIssueKey(issueKey: String): Sequence<CommitDocument>
+    fun findPullRequestsByIssueKey(issueKey: String): Sequence<PullRequestDocument>
     fun findByIssueKey(issueKey: String): SearchSummary
 
     companion object {

diff --git a/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/VcsManager.kt b/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/VcsManager.kt
@@ -12,16 +12,16 @@ import org.octopusden.octopus.vcsfacade.client.common.dto.SearchSummary
 import org.octopusden.octopus.vcsfacade.client.common.dto.Tag
 
 interface VcsManager {
-    fun getTags(sshUrl: String): List<Tag>
-    fun getCommits(sshUrl: String, fromHashOrRef: String?, fromDate: Date?, toHashOrRef: String): List<Commit>
-    fun getCommitsWithFiles(sshUrl: String, fromHashOrRef: String?, fromDate: Date?, toHashOrRef: String): List<CommitWithFiles>
+    fun getTags(sshUrl: String): Sequence<Tag>
+    fun getCommits(sshUrl: String, fromHashOrRef: String?, fromDate: Date?, toHashOrRef: String): Sequence<Commit>
+    fun getCommitsWithFiles(sshUrl: String, fromHashOrRef: String?, fromDate: Date?, toHashOrRef: String): Sequence<CommitWithFiles>
     fun getCommit(sshUrl: String, hashOrRef: String): Commit
     fun getCommitWithFiles(sshUrl: String, hashOrRef: String): CommitWithFiles
     fun createPullRequest(sshUrl: String, createPullRequest: CreatePullRequest): PullRequest
     fun searchIssuesInRanges(searchRequest: SearchIssuesInRangesRequest): SearchIssueInRangesResponse
-    fun findBranches(issueKey: String): List<Branch>
-    fun findCommits(issueKey: String): List<Commit>
-    fun findCommitsWithFiles(issueKey: String): List<CommitWithFiles>
-    fun findPullRequests(issueKey: String): List<PullRequest>
+    fun findBranches(issueKey: String): Sequence<Branch>
+    fun findCommits(issueKey: String): Sequence<Commit>
+    fun findCommitsWithFiles(issueKey: String): Sequence<CommitWithFiles>
+    fun findPullRequests(issueKey: String): Sequence<PullRequest>
     fun find(issueKey: String): SearchSummary
 }
diff --git a/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/VcsService.kt b/server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/VcsService.kt
@@ -19,18 +19,18 @@ abstract class VcsService(vcsProperties: VcsConfig.VcsProperties) {
     fun parse(sshUrl: String) =
         sshUrlRegex.find(sshUrl.lowercase())!!.destructured.let { it.component1().trimEnd('/') to it.component2() }
 
-    abstract fun getBranches(group: String, repository: String): List<Branch>
-    abstract fun getTags(group: String, repository: String): List<Tag>
-    abstract fun getCommits(group: String, repository: String, from: HashOrRefOrDate<String, Date>?, toHashOrRef: String): List<Commit>
-    abstract fun getCommitsWithFiles(group: String, repository: String, from: HashOrRefOrDate<String, Date>?, toHashOrRef: String): List<CommitWithFiles>
+    abstract fun getBranches(group: String, repository: String): Sequence<Branch>
+    abstract fun getTags(group: String, repository: String): Sequence<Tag>
+    abstract fun getCommits(group: String, repository: String, from: HashOrRefOrDate<String, Date>?, toHashOrRef: String): Sequence<Commit>
+    abstract fun getCommitsWithFiles(group: String, repository: String, from: HashOrRefOrDate<String, Date>?, toHashOrRef: String): Sequence<CommitWithFiles>
     abstract fun getCommit(group: String, repository: String, hashOrRef: String): Commit
     abstract fun getCommitWithFiles(group: String, repository: String, hashOrRef: String): CommitWithFiles
     abstract fun createPullRequest(group: String, repository: String, createPullRequest: CreatePullRequest): PullRequest
     abstract fun getPullRequest(group: String, repository: String, index: Long): PullRequest
-    abstract fun findCommits(group: String, repository: String, hashes: Set<String>): List<Commit>
-    abstract fun findPullRequests(group: String, repository: String, indexes: Set<Long>): List<PullRequest>
-    abstract fun findBranches(issueKey: String): List<Branch>
-    abstract fun findCommits(issueKey: String): List<Commit>
-    abstract fun findCommitsWithFiles(issueKey: String): List<CommitWithFiles>
-    abstract fun findPullRequests(issueKey: String): List<PullRequest>
+    abstract fun findCommits(group: String, repository: String, hashes: Set<String>): Sequence<Commit>
+    abstract fun findPullRequests(group: String, repository: String, indexes: Set<Long>): Sequence<PullRequest>
+    abstract fun findBranches(issueKey: String): Sequence<Branch>
+    abstract fun findCommits(issueKey: String): Sequence<Commit>
+    abstract fun findCommitsWithFiles(issueKey: String): Sequence<CommitWithFiles>
+    abstract fun findPullRequests(issueKey: String): Sequence<PullRequest>
 }