Skip to content

Commit

Permalink
Add vcs-facade.master property. (#34)
Browse files Browse the repository at this point in the history
  • Loading branch information
CamaradeRoman authored Apr 19, 2024
1 parent 8d03974 commit a9149fe
Show file tree
Hide file tree
Showing 24 changed files with 90 additions and 73 deletions.
5 changes: 5 additions & 0 deletions ft/docker/bitbucket/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ services:
JDBC_USER: bitbucket
JDBC_PASSWORD: bitbucket
JDBC_URL: jdbc:postgresql://bitbucket-db:5432/bitbucket
healthcheck:
test: curl -s -f bitbucket:7990/status >/dev/null || exit 1
interval: 30s
timeout: 10s
retries: 5

bitbucket-db:
container_name: vcs-facade-ft-bitbucket-db
Expand Down
2 changes: 0 additions & 2 deletions ft/docker/gitlab/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,3 @@ services:
POSTGRES_USER: gitlab
POSTGRES_PASSWORD: gitlab
POSTGRES_DB: gitlab
ports:
- "6432:5432"
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import org.octopusden.octopus.vcsfacade.client.common.exception.NotFoundExceptio
import org.octopusden.octopus.vcsfacade.client.impl.ClassicVcsFacadeClient
import org.octopusden.octopus.vcsfacade.client.impl.VcsFacadeClientParametersProvider

abstract class BaseVcsFacadeFuncTest(testClient: TestClient, sshUrlFormat: String) :
abstract class BaseVcsFacadeFunctionalTest(testClient: TestClient, sshUrlFormat: String) :
BaseVcsFacadeTest(testClient, sshUrlFormat) {

override fun requestTags(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import org.octopusden.octopus.infastructure.bitbucket.test.BitbucketTestClient
private const val VCS_HOST = "bitbucket:7990"

@EnabledIfSystemProperty(named = "test.profile", matches = "bitbucket")
class VcsFacadeBitbucketTest : BaseVcsFacadeFuncTest(
class VcsFacadeFunctionalTestBitbucket : BaseVcsFacadeFunctionalTest(
BitbucketTestClient("http://localhost:7990", BITBUCKET_USER, BITBUCKET_PASSWORD, VCS_HOST),
"ssh://git@$VCS_HOST/%s/%s.git"
) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import org.octopusden.octopus.infrastructure.gitea.test.GiteaTestClient
private const val VCS_HOST = "gitea:3000"

@EnabledIfSystemProperty(named = "test.profile", matches = "gitea")
class VcsFacadeGiteaTest : BaseVcsFacadeFuncTest(
class VcsFacadeFunctionalTestGitea : BaseVcsFacadeFunctionalTest(
GiteaTestClient("http://localhost:3000", GITEA_USER, GITEA_PASSWORD, VCS_HOST),
"ssh://git@$VCS_HOST:%s/%s.git"
) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import org.octopusden.octopus.infrastructure.gitlab.test.GitlabTestClient
private const val VCS_HOST = "gitlab:8990"

@EnabledIfSystemProperty(named = "test.profile", matches = "gitlab")
class VcsFacadeGitlabTest : BaseVcsFacadeFuncTest(
class VcsFacadeFunctionalTestGitlab : BaseVcsFacadeFunctionalTest(
GitlabTestClient("http://localhost:8990", GITLAB_USER, GITLAB_PASSWORD, VCS_HOST),
"ssh://git@$VCS_HOST:%s/%s.git"
) {
Expand Down
5 changes: 5 additions & 0 deletions server/docker/bitbucket/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ services:
JDBC_USER: bitbucket
JDBC_PASSWORD: bitbucket
JDBC_URL: jdbc:postgresql://bitbucket-db:5432/bitbucket
healthcheck:
test: curl -s -f bitbucket:7990/status >/dev/null || exit 1
interval: 30s
timeout: 10s
retries: 5

bitbucket-db:
container_name: vcs-facade-ut-bitbucket-db
Expand Down
2 changes: 0 additions & 2 deletions server/docker/gitlab/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,3 @@ services:
POSTGRES_USER: gitlab
POSTGRES_PASSWORD: gitlab
POSTGRES_DB: gitlab
ports:
- "6432:5432"
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration

@Configuration
class VCSConfig(val giteaProperties: GiteaProperties?) {
class VcsConfig(val giteaProperties: GiteaProperties?) {
@ConfigurationProperties("vcs-facade.vcs.bitbucket")
@ConditionalOnProperty(
prefix = "vcs-facade.vcs.bitbucket", name = ["enabled"], havingValue = "true", matchIfMissing = true
)
class BitbucketProperties(
host: String, token: String?, username: String?, password: String?, healthCheck: HealthCheck
) : VCSProperties(
) : VcsProperties(
host,
if (token?.isNotBlank() == true) token else null,
if (token?.isNotBlank() == true) null else username,
Expand All @@ -32,7 +32,7 @@ class VCSConfig(val giteaProperties: GiteaProperties?) {
password: String?,
healthCheck: HealthCheck,
val index: GiteaIndexProperties?
) : VCSProperties(
) : VcsProperties(
host,
if (token?.isNotBlank() == true) token else null,
if (token?.isNotBlank() == true) null else username,
Expand All @@ -44,9 +44,9 @@ class VCSConfig(val giteaProperties: GiteaProperties?) {
@ConditionalOnProperty(
prefix = "vcs-facade.vcs.gitlab", name = ["enabled"], havingValue = "true", matchIfMissing = true
)
class GitLabProperties(
class GitlabProperties(
host: String, token: String?, username: String?, password: String?, healthCheck: HealthCheck
) : VCSProperties(
) : VcsProperties(
host,
if (token?.isNotBlank() == true) token else null,
if (token?.isNotBlank() == true) null else username,
Expand All @@ -58,7 +58,7 @@ class VCSConfig(val giteaProperties: GiteaProperties?) {

data class GiteaIndexScanProperties(val cron: String?, val executor: ExecutorProperties?)

abstract class VCSProperties(
abstract class VcsProperties(
val host: String, val token: String?, val username: String?, val password: String?, val healthCheck: HealthCheck
)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
package org.octopusden.octopus.vcsfacade.config

import java.net.InetAddress
import org.springframework.beans.factory.annotation.Value
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.web.servlet.config.annotation.ViewControllerRegistry
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer

@Configuration
class WebConfig : WebMvcConfigurer {
class WebConfig(
@Value("\${vcs-facade.master:}") val master: String
) : WebMvcConfigurer {
override fun addViewControllers(registry: ViewControllerRegistry) {
registry.addRedirectViewController("/", "swagger-ui/index.html")
}

@Bean
fun isMaster() = master.isBlank() || master.equals(InetAddress.getLocalHost().hostName, true)
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import jakarta.servlet.http.HttpServletRequest
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec
import org.octopusden.octopus.vcsfacade.config.VCSConfig
import org.octopusden.octopus.vcsfacade.config.VcsConfig
import org.octopusden.octopus.vcsfacade.dto.GiteaCreateRefEvent
import org.octopusden.octopus.vcsfacade.dto.GiteaDeleteRefEvent
import org.octopusden.octopus.vcsfacade.dto.GiteaPullRequestEvent
Expand All @@ -28,7 +28,7 @@ import org.springframework.web.bind.annotation.RestController
prefix = "vcs-facade", name = ["vcs.gitea.enabled", "opensearch.enabled"], havingValue = "true", matchIfMissing = true
)
class GiteaIndexerController(
giteaProperties: VCSConfig.GiteaProperties,
giteaProperties: VcsConfig.GiteaProperties,
private val giteaIndexerService: GiteaIndexerService,
private val objectMapper: ObjectMapper
) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import org.octopusden.octopus.vcsfacade.config.JobConfig
import org.octopusden.octopus.vcsfacade.dto.RepositoryResponse
import org.octopusden.octopus.vcsfacade.exception.JobProcessingException
import org.octopusden.octopus.vcsfacade.issue.IssueKeyParser
import org.octopusden.octopus.vcsfacade.service.VCSManager
import org.octopusden.octopus.vcsfacade.service.VcsManager
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.core.task.AsyncTaskExecutor
Expand All @@ -37,7 +37,7 @@ import org.springframework.web.bind.annotation.RestController
@RequestMapping("rest/api/1/repository")
class RepositoryController(
private val jobProperties: JobConfig.JobProperties,
private val vcsManager: VCSManager,
private val vcsManager: VcsManager,
@Qualifier("jobExecutor") private val jobExecutor: AsyncTaskExecutor
) {
private val requestJobs = ConcurrentHashMap<String, Future<out VcsFacadeResponse>>()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import org.octopusden.octopus.vcsfacade.config.JobConfig
import org.octopusden.octopus.vcsfacade.dto.RepositoryResponse
import org.octopusden.octopus.vcsfacade.exception.JobProcessingException
import org.octopusden.octopus.vcsfacade.issue.IssueKeyParser
import org.octopusden.octopus.vcsfacade.service.VCSManager
import org.octopusden.octopus.vcsfacade.service.VcsManager
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.core.task.AsyncTaskExecutor
Expand All @@ -45,7 +45,7 @@ import org.springframework.web.bind.annotation.RestController
@Deprecated(message = "Deprecated controller", replaceWith = ReplaceWith("RepositoryController"))
class RepositoryControllerOld(
private val jobProperties: JobConfig.JobProperties,
private val vcsManager: VCSManager,
private val vcsManager: VcsManager,
@Qualifier("jobExecutor") private val jobExecutor: AsyncTaskExecutor
) {
private val requestJobs = ConcurrentHashMap<String, Future<out VcsFacadeResponse>>()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import org.octopusden.octopus.vcsfacade.client.common.dto.SearchIssuesInRangesRe
import org.octopusden.octopus.vcsfacade.client.common.dto.SearchSummary
import org.octopusden.octopus.vcsfacade.client.common.dto.Tag

interface VCSManager {
interface VcsManager {
fun getTags(sshUrl: String): List<Tag>
fun getCommits(sshUrl: String, fromId: String?, fromDate: Date?, toId: String): List<Commit>
fun getCommit(sshUrl: String, id: String): Commit
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ import org.octopusden.octopus.vcsfacade.client.common.dto.Commit
import org.octopusden.octopus.vcsfacade.client.common.dto.CreatePullRequest
import org.octopusden.octopus.vcsfacade.client.common.dto.PullRequest
import org.octopusden.octopus.vcsfacade.client.common.dto.Tag
import org.octopusden.octopus.vcsfacade.config.VCSConfig
import org.octopusden.octopus.vcsfacade.config.VcsConfig

abstract class VCSService(vcsProperties: VCSConfig.VCSProperties) {
abstract class VcsService(vcsProperties: VcsConfig.VcsProperties) {
protected val httpUrl = vcsProperties.host.lowercase().trimEnd('/')
protected val host = httpUrl.replace("^(https|http)://".toRegex(), "")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import org.octopusden.octopus.infrastructure.bitbucket.client.BitbucketCredentia
import org.octopusden.octopus.infrastructure.bitbucket.client.createPullRequestWithDefaultReviewers
import org.octopusden.octopus.infrastructure.bitbucket.client.dto.BitbucketBranch
import org.octopusden.octopus.infrastructure.bitbucket.client.dto.BitbucketCommit
import org.octopusden.octopus.infrastructure.bitbucket.client.dto.BitbucketLinkName
import org.octopusden.octopus.infrastructure.bitbucket.client.dto.BitbucketPullRequest
import org.octopusden.octopus.infrastructure.bitbucket.client.dto.BitbucketPullRequestState
import org.octopusden.octopus.infrastructure.bitbucket.client.dto.BitbucketTag
Expand All @@ -29,8 +28,8 @@ import org.octopusden.octopus.vcsfacade.client.common.dto.PullRequestStatus
import org.octopusden.octopus.vcsfacade.client.common.dto.Repository
import org.octopusden.octopus.vcsfacade.client.common.dto.Tag
import org.octopusden.octopus.vcsfacade.client.common.dto.User
import org.octopusden.octopus.vcsfacade.config.VCSConfig
import org.octopusden.octopus.vcsfacade.service.VCSService
import org.octopusden.octopus.vcsfacade.config.VcsConfig
import org.octopusden.octopus.vcsfacade.service.VcsService
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.stereotype.Service
Expand All @@ -40,8 +39,8 @@ import org.springframework.stereotype.Service
prefix = "vcs-facade.vcs.bitbucket", name = ["enabled"], havingValue = "true", matchIfMissing = true
)
class BitbucketService(
bitbucketProperties: VCSConfig.BitbucketProperties,
) : VCSService(bitbucketProperties) {
bitbucketProperties: VcsConfig.BitbucketProperties,
) : VcsService(bitbucketProperties) {
private val bitbucketClient: BitbucketClient = BitbucketClassicClient(object : BitbucketClientParametersProvider {
override fun getApiUrl(): String = httpUrl

Expand Down Expand Up @@ -151,9 +150,8 @@ class BitbucketService(

override fun findCommits(issueKey: String): List<Commit> {
log.trace("=> findCommits({})", issueKey)
return bitbucketClient.getCommits(issueKey).map { bitbucketJiraCommit ->
val (group, repository) = parse(bitbucketJiraCommit.repository.links.clone.find { it.name == BitbucketLinkName.SSH }!!.href)
bitbucketJiraCommit.toCommit.toCommit(group, repository)
return bitbucketClient.getCommits(issueKey).map {
it.toCommit.toCommit(it.repository.project.key.lowercase(), it.repository.slug.lowercase())
}.also {
log.trace("<= findCommits({}): {}", issueKey, it)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ import org.octopusden.octopus.vcsfacade.dto.IndexReport
import org.octopusden.octopus.vcsfacade.dto.VcsServiceType.GITEA
import org.octopusden.octopus.vcsfacade.service.GiteaIndexerService
import org.octopusden.octopus.vcsfacade.service.OpenSearchService
import org.octopusden.octopus.vcsfacade.service.OpenSearchService.Companion.toDocument
import org.octopusden.octopus.vcsfacade.service.OpenSearchService.Companion.toDto
import org.octopusden.octopus.vcsfacade.service.impl.GiteaService.Companion.toBranch
import org.octopusden.octopus.vcsfacade.service.impl.GiteaService.Companion.toOrganizationAndRepository
import org.octopusden.octopus.vcsfacade.service.impl.GiteaService.Companion.toPullRequest
import org.octopusden.octopus.vcsfacade.service.impl.GiteaService.Companion.toTag
import org.octopusden.octopus.vcsfacade.service.OpenSearchService.Companion.toDocument
import org.octopusden.octopus.vcsfacade.service.OpenSearchService.Companion.toDto
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
Expand All @@ -39,19 +39,24 @@ import org.springframework.stereotype.Service
class GiteaIndexerServiceImpl(
private val giteaService: GiteaService,
private val openSearchService: OpenSearchService,
@Qualifier("giteaIndexScanExecutor") private val giteaIndexScanExecutor: AsyncTaskExecutor
@Qualifier("giteaIndexScanExecutor") private val giteaIndexScanExecutor: AsyncTaskExecutor,
@Qualifier("isMaster") private val isMaster: Boolean
) : GiteaIndexerService {

@Scheduled(cron = "#{ @giteaIndexScanCron }")
private fun scan() = try {
log.trace("Submitting {} repositories for scan", GITEA)
val repositories = openSearchService.getRepositories(GITEA) + giteaService.getRepositories()
.map { it.sshUrl.toRepositoryDocument() }
log.trace("Repositories collected for scan: {}", repositories)
repositories.forEach { submitRepositoryScan(it) }
log.debug("Submitted {} {} repositories for scan", repositories.size, GITEA)
} catch (e: Exception) {
log.error("Unable to submit $GITEA repositories for scan", e)
private fun scan() = if (isMaster) {
try {
log.trace("Submitting {} repositories for scan", GITEA)
val repositories = openSearchService.getRepositories(GITEA) + giteaService.getRepositories()
.map { it.sshUrl.toRepositoryDocument() }
log.trace("Repositories collected for scan: {}", repositories)
repositories.forEach { submitRepositoryScan(it) }
log.debug("Submitted {} {} repositories for scan", repositories.size, GITEA)
} catch (e: Exception) {
log.error("Unable to submit $GITEA repositories for scan", e)
}
} else {
log.debug("Suppress {} gitea repositories scan on non-master instance", GITEA)
}

override fun registerGiteaCreateRefEvent(giteaCreateRefEvent: GiteaCreateRefEvent) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ import org.octopusden.octopus.vcsfacade.client.common.dto.PullRequestStatus
import org.octopusden.octopus.vcsfacade.client.common.dto.Repository
import org.octopusden.octopus.vcsfacade.client.common.dto.Tag
import org.octopusden.octopus.vcsfacade.client.common.dto.User
import org.octopusden.octopus.vcsfacade.config.VCSConfig
import org.octopusden.octopus.vcsfacade.service.VCSService
import org.octopusden.octopus.vcsfacade.config.VcsConfig
import org.octopusden.octopus.vcsfacade.service.VcsService
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.stereotype.Service
Expand All @@ -44,8 +44,8 @@ import org.springframework.stereotype.Service
prefix = "vcs-facade.vcs.gitea", name = ["enabled"], havingValue = "true", matchIfMissing = true
)
class GiteaService(
giteaProperties: VCSConfig.GiteaProperties
) : VCSService(giteaProperties) {
giteaProperties: VcsConfig.GiteaProperties
) : VcsService(giteaProperties) {
private val client: GiteaClient = GiteaClassicClient(object : ClientParametersProvider {
override fun getApiUrl() = httpUrl

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ import org.octopusden.octopus.vcsfacade.client.common.dto.Repository
import org.octopusden.octopus.vcsfacade.client.common.dto.Tag
import org.octopusden.octopus.vcsfacade.client.common.dto.User
import org.octopusden.octopus.vcsfacade.client.common.exception.NotFoundException
import org.octopusden.octopus.vcsfacade.config.VCSConfig
import org.octopusden.octopus.vcsfacade.service.VCSService
import org.octopusden.octopus.vcsfacade.config.VcsConfig
import org.octopusden.octopus.vcsfacade.service.VcsService
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.stereotype.Service
Expand All @@ -34,16 +34,16 @@ import org.gitlab4j.api.models.Tag as GitlabTag
prefix = "vcs-facade.vcs.gitlab", name = ["enabled"], havingValue = "true", matchIfMissing = true
)
class GitlabService(
gitLabProperties: VCSConfig.GitLabProperties
) : VCSService(gitLabProperties) {
gitlabProperties: VcsConfig.GitlabProperties
) : VcsService(gitlabProperties) {
private val clientFunc: () -> GitLabApi = {
val authException by lazy {
IllegalStateException("Auth Token or username/password must be specified for Gitlab access")
}
gitLabProperties.token?.let { GitLabApi(httpUrl, gitLabProperties.token) } ?: getGitlabApi(
gitLabProperties, authException
gitlabProperties.token?.let { GitLabApi(httpUrl, gitlabProperties.token) } ?: getGitlabApi(
gitlabProperties, authException
).also { api ->
api.setAuthTokenSupplier { getToken(gitLabProperties, authException) }
api.setAuthTokenSupplier { getToken(gitlabProperties, authException) }
}
}

Expand Down Expand Up @@ -264,22 +264,22 @@ class GitlabService(
}

private fun getGitlabApi(
gitLabProperties: VCSConfig.GitLabProperties, authException: IllegalStateException
gitlabProperties: VcsConfig.GitlabProperties, authException: IllegalStateException
) = GitLabApi.oauth2Login(
gitLabProperties.host,
gitLabProperties.username ?: throw authException,
gitLabProperties.password ?: throw authException
gitlabProperties.host,
gitlabProperties.username ?: throw authException,
gitlabProperties.password ?: throw authException
).also { api ->
tokenObtained = Instant.now()
token = api.authToken
}

private fun getToken(
gitLabProperties: VCSConfig.GitLabProperties, authException: IllegalStateException
gitlabProperties: VcsConfig.GitlabProperties, authException: IllegalStateException
): String {
if (tokenObtained.isBefore(Instant.now().minus(Duration.ofMinutes(110)))) {
log.info("Refresh auth token")
getGitlabApi(gitLabProperties, authException)
getGitlabApi(gitlabProperties, authException)
}
return token
}
Expand Down
Loading

0 comments on commit a9149fe

Please sign in to comment.