Add totalFiles counter and commitFilesLimit parameter. (#39)

Also:
* Reduce batch limit from 1000 to 100.
* Add fallback for saving batches.
* Use new version of gitea client to fix pull-request deserialization issue.

client/src/main/kotlin/org/octopusden/octopus/vcsfacade/client/VcsFacadeClient.kt

@@ -23,19 +23,24 @@ interface VcsFacadeClient {
         @Param("toHashOrRef") toHashOrRef: String
     ): List<Commit>
 
-    @RequestLine("GET rest/api/2/repository/commits/files?sshUrl={sshUrl}&fromHashOrRef={fromHashOrRef}&fromDate={fromDate}&toHashOrRef={toHashOrRef}")
+    @RequestLine("GET rest/api/2/repository/commits/files?sshUrl={sshUrl}&fromHashOrRef={fromHashOrRef}&fromDate={fromDate}&toHashOrRef={toHashOrRef}&commitFilesLimit={commitFilesLimit}")
     fun getCommitsWithFiles(
         @Param("sshUrl") sshUrl: String,
         @Param("fromHashOrRef") fromHashOrRef: String?,
         @Param("fromDate", expander = DateToISOExpander::class) fromDate: Date?,
-        @Param("toHashOrRef") toHashOrRef: String
+        @Param("toHashOrRef") toHashOrRef: String,
+        @Param("commitFilesLimit") commitFilesLimit: Int?
     ): List<CommitWithFiles>
 
     @RequestLine("GET rest/api/2/repository/commit?sshUrl={sshUrl}&hashOrRef={hashOrRef}")
     fun getCommit(@Param("sshUrl") sshUrl: String, @Param("hashOrRef") hashOrRef: String): Commit
 
-    @RequestLine("GET rest/api/2/repository/commit/files?sshUrl={sshUrl}&hashOrRef={hashOrRef}")
-    fun getCommitWithFiles(@Param("sshUrl") sshUrl: String, @Param("hashOrRef") hashOrRef: String): CommitWithFiles
+    @RequestLine("GET rest/api/2/repository/commit/files?sshUrl={sshUrl}&hashOrRef={hashOrRef}&commitFilesLimit={commitFilesLimit}")
+    fun getCommitWithFiles(
+        @Param("sshUrl") sshUrl: String,
+        @Param("hashOrRef") hashOrRef: String,
+        @Param("commitFilesLimit") commitFilesLimit: Int?
+    ): CommitWithFiles
 
     @RequestLine("GET rest/api/2/repository/issues?sshUrl={sshUrl}&fromHashOrRef={fromHashOrRef}&fromDate={fromDate}&toHashOrRef={toHashOrRef}")
     fun getIssuesFromCommits(
@@ -65,8 +70,11 @@ interface VcsFacadeClient {
     @RequestLine("GET rest/api/2/repository/find/{issueKey}/commits")
     fun findCommitsByIssueKey(@Param("issueKey") issueKey: String): List<Commit>
 
-    @RequestLine("GET rest/api/2/repository/find/{issueKey}/commits/files")
-    fun findCommitsWithFilesByIssueKey(@Param("issueKey") issueKey: String): List<CommitWithFiles>
+    @RequestLine("GET rest/api/2/repository/find/{issueKey}/commits/files?commitFilesLimit={commitFilesLimit}")
+    fun findCommitsWithFilesByIssueKey(
+        @Param("issueKey") issueKey: String,
+        @Param("commitFilesLimit") commitFilesLimit: Int?
+    ): List<CommitWithFiles>
 
     @RequestLine("GET rest/api/2/repository/find/{issueKey}/pull-requests")
     fun findPullRequestsByIssueKey(@Param("issueKey") issueKey: String): List<PullRequest>

client/src/main/kotlin/org/octopusden/octopus/vcsfacade/client/impl/ClassicVcsFacadeClient.kt

@@ -32,12 +32,18 @@ class ClassicVcsFacadeClient(
     override fun getCommits(sshUrl: String, fromHashOrRef: String?, fromDate: Date?, toHashOrRef: String) =
         client.getCommits(sshUrl, fromHashOrRef, fromDate, toHashOrRef)
 
-    override fun getCommitsWithFiles(sshUrl: String, fromHashOrRef: String?, fromDate: Date?, toHashOrRef: String) =
-        client.getCommitsWithFiles(sshUrl, fromHashOrRef, fromDate, toHashOrRef)
+    override fun getCommitsWithFiles(
+        sshUrl: String,
+        fromHashOrRef: String?,
+        fromDate: Date?,
+        toHashOrRef: String,
+        commitFilesLimit: Int?
+    ) = client.getCommitsWithFiles(sshUrl, fromHashOrRef, fromDate, toHashOrRef, commitFilesLimit)
 
     override fun getCommit(sshUrl: String, hashOrRef: String) = client.getCommit(sshUrl, hashOrRef)
 
-    override fun getCommitWithFiles(sshUrl: String, hashOrRef: String) = client.getCommitWithFiles(sshUrl, hashOrRef)
+    override fun getCommitWithFiles(sshUrl: String, hashOrRef: String, commitFilesLimit: Int?) =
+        client.getCommitWithFiles(sshUrl, hashOrRef, commitFilesLimit)
 
     override fun getIssuesFromCommits(sshUrl: String, fromHashOrRef: String?, fromDate: Date?, toHashOrRef: String) =
         client.getIssuesFromCommits(sshUrl, fromHashOrRef, fromDate, toHashOrRef)
@@ -56,7 +62,8 @@ class ClassicVcsFacadeClient(
 
     override fun findCommitsByIssueKey(issueKey: String) = client.findCommitsByIssueKey(issueKey)
 
-    override fun findCommitsWithFilesByIssueKey(issueKey: String) = client.findCommitsWithFilesByIssueKey(issueKey)
+    override fun findCommitsWithFilesByIssueKey(issueKey: String, commitFilesLimit: Int?) =
+        client.findCommitsWithFilesByIssueKey(issueKey, commitFilesLimit)
 
     override fun findPullRequestsByIssueKey(issueKey: String) = client.findPullRequestsByIssueKey(issueKey)
 

common/src/main/kotlin/org/octopusden/octopus/vcsfacade/client/common/dto/CommitWithFiles.kt

@@ -2,6 +2,7 @@ package org.octopusden.octopus.vcsfacade.client.common.dto
 
 data class CommitWithFiles(
     val commit: Commit,
+    val totalFiles: Int,
     val files: List<FileChange>
 ) : VcsFacadeResponse, Comparable<CommitWithFiles> {
     override fun compareTo(other: CommitWithFiles) = commit compareTo other.commit

gradle.properties

@@ -6,7 +6,7 @@ bmuschko-docker-plugin.version=9.4.0
 docker-compose-plugin.version=0.16.9
 spring-data-opensearch.version=1.3.0
 springdoc-openapi.version=2.3.0
-external-systems-client.version=2.0.38
+external-systems-client.version=2.0.39
 gitlab4j-api.version=6.0.0-rc.4
 bitbucket.version=8.14.0-jdk11
 gitea.version=1.21.10

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/controller/RepositoryController.kt

@@ -8,7 +8,6 @@ import java.util.concurrent.ExecutionException
 import java.util.concurrent.Future
 import java.util.concurrent.TimeUnit
 import org.octopusden.octopus.vcsfacade.client.common.Constants
-import org.octopusden.octopus.vcsfacade.client.common.dto.Commit
 import org.octopusden.octopus.vcsfacade.client.common.dto.CommitWithFiles
 import org.octopusden.octopus.vcsfacade.client.common.dto.CreatePullRequest
 import org.octopusden.octopus.vcsfacade.client.common.dto.PullRequest
@@ -66,15 +65,20 @@ class RepositoryController(
         @RequestParam("fromHashOrRef", required = false) fromHashOrRef: String?,
         @RequestParam("fromDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) fromDate: Date?,
         @RequestParam("toHashOrRef") toHashOrRef: String,
+        @RequestParam("commitFilesLimit", defaultValue = "0") commitFilesLimit: Int,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
         log.info(
-            "Get commits ({},{}] with files in `{}` repository",
+            "Get commits ({},{}] with files (limit {}) in `{}` repository",
             (fromHashOrRef ?: fromDate?.toString()).orEmpty(),
             toHashOrRef,
+            commitFilesLimit,
             sshUrl
         )
-        RepositoryResponse(vcsManager.getCommitsWithFiles(sshUrl, fromHashOrRef, fromDate, toHashOrRef))
+        RepositoryResponse(
+            vcsManager.getCommitsWithFiles(sshUrl, fromHashOrRef, fromDate, toHashOrRef)
+                .map { it.applyCommitFilesLimit(commitFilesLimit) }
+        )
     }.data.sorted()
 
     @GetMapping("commit", produces = [MediaType.APPLICATION_JSON_VALUE])
@@ -91,10 +95,11 @@ class RepositoryController(
     fun getCommitWithFiles(
         @RequestParam("sshUrl") sshUrl: String,
         @RequestParam("hashOrRef") hashOrRef: String,
+        @RequestParam("commitFilesLimit", defaultValue = "0") commitFilesLimit: Int,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Get commit {} in `{}` repository", hashOrRef, sshUrl)
-        vcsManager.getCommitWithFiles(sshUrl, hashOrRef)
+        log.info("Get commit {} with files (limit {}) in `{}` repository", hashOrRef, commitFilesLimit, sshUrl)
+        vcsManager.getCommitWithFiles(sshUrl, hashOrRef).applyCommitFilesLimit(commitFilesLimit)
     }
 
     @GetMapping("issues", produces = [MediaType.APPLICATION_JSON_VALUE])
@@ -184,10 +189,13 @@ class RepositoryController(
     @GetMapping("find/{issueKey}/commits/files", produces = [MediaType.APPLICATION_JSON_VALUE])
     fun findCommitsWithFilesByIssueKey(
         @PathVariable("issueKey") issueKey: String,
+        @RequestParam("commitFilesLimit", defaultValue = "0") commitFilesLimit: Int,
         @RequestHeader(Constants.DEFERRED_RESULT_HEADER, required = false) requestId: String?
     ) = processRequest(requestId ?: UUID.randomUUID().toString()) {
-        log.info("Find commits with files by issue key {}", issueKey)
-        RepositoryResponse(vcsManager.findCommitsWithFiles(issueKey))
+        log.info("Find commits with files (limit {}) by issue key {}", commitFilesLimit, issueKey)
+        RepositoryResponse(
+            vcsManager.findCommitsWithFiles(issueKey).map { it.applyCommitFilesLimit(commitFilesLimit) }
+        )
     }.data.sorted()
 
     @GetMapping("find/{issueKey}/pull-requests", produces = [MediaType.APPLICATION_JSON_VALUE])
@@ -232,5 +240,9 @@ class RepositoryController(
 
     companion object {
         private val log = LoggerFactory.getLogger(RepositoryController::class.java)
+
+        private fun CommitWithFiles.applyCommitFilesLimit(commitFilesLimit: Int) = if (commitFilesLimit > 0)
+            CommitWithFiles(commit, totalFiles, files.take(commitFilesLimit))
+        else this
     }
 }

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/document/CommitDocument.kt

@@ -20,6 +20,7 @@ class CommitDocument(
     @Field(type = FieldType.Object) val author: UserDocument,
     @Field(type = FieldType.Keyword) val parents: List<String>,
     @Field(type = FieldType.Keyword) val link: String,
+    //TODO: Sometimes a commit contains enormous number of affected files. Is it better to store files in separate index?
     @Field(type = FieldType.Object) val files: List<FileChangeDocument>
 ) : BaseDocument(id(repository.id, hash)) {
     override fun toString() =

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/repository/CommitRepository.kt

@@ -9,6 +9,6 @@ import org.springframework.data.repository.CrudRepository
 )
 interface CommitRepository : CrudRepository<CommitDocument, String> {
     fun searchByMessageContaining(messageToken: String): List<CommitDocument>
-    fun searchFirst1000ByRepositoryIdAndIdAfterOrderByIdAsc(repositoryId: String, id: String): List<CommitDocument>
+    fun searchFirst100ByRepositoryIdAndIdAfterOrderByIdAsc(repositoryId: String, id: String): List<CommitDocument>
     fun deleteByRepositoryId(repositoryId: String)
 }

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/repository/PullRequestRepository.kt

@@ -9,6 +9,6 @@ import org.springframework.data.repository.CrudRepository
 )
 interface PullRequestRepository : CrudRepository<PullRequestDocument, String> {
     fun searchByTitleContainingOrDescriptionContaining(titleToken: String, descriptionToken: String): List<PullRequestDocument>
-    fun searchFirst1000ByRepositoryIdAndIdAfterOrderByIdAsc(repositoryId: String, id: String): List<PullRequestDocument>
+    fun searchFirst100ByRepositoryIdAndIdAfterOrderByIdAsc(repositoryId: String, id: String): List<PullRequestDocument>
     fun deleteByRepositoryId(repositoryId: String)
 }

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/repository/RefRepository.kt

@@ -10,6 +10,6 @@ import org.springframework.data.repository.CrudRepository
 )
 interface RefRepository : CrudRepository<RefDocument, String> {
     fun searchByTypeAndNameContaining(type: RefType, nameToken: String): List<RefDocument>
-    fun searchFirst1000ByRepositoryIdAndIdAfterOrderByIdAsc(repositoryId: String, id: String): List<RefDocument>
+    fun searchFirst100ByRepositoryIdAndIdAfterOrderByIdAsc(repositoryId: String, id: String): List<RefDocument>
     fun deleteByRepositoryId(repositoryId: String)
 }

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/repository/RepositoryInfoRepository.kt

@@ -9,5 +9,5 @@ import org.springframework.data.repository.CrudRepository
     prefix = "vcs-facade.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = true
 )
 interface RepositoryInfoRepository : CrudRepository<RepositoryInfoDocument, String> {
-    fun searchFirst1000ByRepositoryTypeAndIdAfterOrderByIdAsc(type: VcsServiceType, id: String): List<RepositoryInfoDocument>
+    fun searchFirst100ByRepositoryTypeAndIdAfterOrderByIdAsc(type: VcsServiceType, id: String): List<RepositoryInfoDocument>
 }

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/OpenSearchService.kt

@@ -67,6 +67,7 @@ interface OpenSearchService {
 
         fun CommitDocument.toDto() = CommitWithFiles(
             Commit(hash, message, date, author.toDto(), parents, link, repository.toDto()),
+            files.size,
             files.map { it.toDto() }
         )
 

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/impl/BitbucketService.kt

@@ -103,7 +103,8 @@ class BitbucketService(
     ): List<CommitWithFiles> {
         log.trace("=> getCommitsWithFiles({}, {}, {}, {})", group, repository, from, toHashOrRef)
         return getCommits(group, repository, from, toHashOrRef).map {
-            CommitWithFiles(it, getCommitChanges(group, repository, it))
+            val fileChanges = getCommitChanges(group, repository, it)
+            CommitWithFiles(it, fileChanges.size, fileChanges)
         }.also { log.trace("<= getCommitsWithFiles({}, {}, {}, {}): {}", group, repository, from, toHashOrRef, it) }
     }
 
@@ -117,7 +118,8 @@ class BitbucketService(
     override fun getCommitWithFiles(group: String, repository: String, hashOrRef: String): CommitWithFiles {
         log.trace("=> getCommitWithFiles({}, {}, {})", group, repository, hashOrRef)
         return with(getCommit(group, repository, hashOrRef)) {
-            CommitWithFiles(this, getCommitChanges(group, repository, this))
+            val fileChanges = getCommitChanges(group, repository, this)
+            CommitWithFiles(this, fileChanges.size, fileChanges)
         }.also { log.trace("<= getCommitWithFiles({}, {}, {}): {}", group, repository, hashOrRef, it) }
     }
 
@@ -190,7 +192,8 @@ class BitbucketService(
             val group = it.repository.project.key.lowercase()
             val repository = it.repository.slug.lowercase()
             val commit = it.toCommit.toCommit(group, repository)
-            CommitWithFiles(commit, getCommitChanges(group, repository, commit))
+            val fileChanges = getCommitChanges(group, repository, commit)
+            CommitWithFiles(commit, fileChanges.size, fileChanges)
         }.also { log.trace("<= findCommitsWithFiles({}): {}", issueKey, it) }
     }
 

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/impl/GiteaService.kt

@@ -278,15 +278,16 @@ class GiteaService(
             BigInteger(1, MessageDigest.getInstance("SHA-1").digest(toByteArray())).toString(16).padStart(32, '0')
 
         private fun GiteaCommit.toCommitWithFiles(repository: Repository) = with(toCommit(repository)) {
-            CommitWithFiles(this, files!!.map {
+            val fileChanges = files!!.map {
                 FileChange(
                     when (it.status) {
                         GiteaCommit.GiteaCommitAffectedFileStatus.ADDED -> FileChangeType.ADD
                         GiteaCommit.GiteaCommitAffectedFileStatus.MODIFIED -> FileChangeType.MODIFY
                         GiteaCommit.GiteaCommitAffectedFileStatus.REMOVED -> FileChangeType.DELETE
                     }, it.filename, "${this.link}#diff-${it.filename.sha1()}"
                 )
-            })
+            }
+            CommitWithFiles(this, fileChanges.size, fileChanges)
         }
 
         fun GiteaPullRequest.toPullRequest(

server/src/main/kotlin/org/octopusden/octopus/vcsfacade/service/impl/GitlabService.kt

@@ -116,7 +116,7 @@ class GitlabService(
         toHashOrRef: String
     ): List<CommitWithFiles> {
         log.warn("There is no native implementation of getCommitsWithFiles")
-        return getCommits(group, repository, from, toHashOrRef).map { CommitWithFiles(it, emptyList()) }
+        return getCommits(group, repository, from, toHashOrRef).map { CommitWithFiles(it, 0, emptyList()) }
     }
 
     override fun getCommit(group: String, repository: String, hashOrRef: String): Commit {
@@ -128,7 +128,7 @@ class GitlabService(
 
     override fun getCommitWithFiles(group: String, repository: String, hashOrRef: String): CommitWithFiles {
         log.warn("There is no native implementation of getCommitWithFiles")
-        return CommitWithFiles(getCommit(group, repository, hashOrRef), emptyList())
+        return CommitWithFiles(getCommit(group, repository, hashOrRef), 0, emptyList())
     }
 
     override fun createPullRequest(