From 2bdd4bacd2d4dd041fcfc5adca7f03f165cf289b Mon Sep 17 00:00:00 2001 From: mghilardelli Date: Thu, 7 Nov 2024 17:19:23 +0100 Subject: [PATCH 1/2] feat: send logs to splunk --- backend/pom.xml | 4 +++ .../domain/logging/ConsoleLogService.kt | 8 ----- .../ch/sbb/backend/domain/logging/LogEntry.kt | 11 +++++-- .../domain/logging/MultiTenantLogService.kt | 31 +++++-------------- .../domain/logging/SplunkLogService.kt | 28 ++++++++++++++--- .../backend/domain/logging/SplunkRequest.kt | 17 ++++++++++ .../domain/tenancy/ConfigTenantService.kt | 2 +- .../configuration/LogDestination.kt | 1 - .../infrastructure/configuration/Tenant.kt | 10 +++--- backend/src/main/resources/application.yaml | 6 +++- 10 files changed, 72 insertions(+), 46 deletions(-) delete mode 100644 backend/src/main/kotlin/ch/sbb/backend/domain/logging/ConsoleLogService.kt create mode 100644 backend/src/main/kotlin/ch/sbb/backend/domain/logging/SplunkRequest.kt diff --git a/backend/pom.xml b/backend/pom.xml index 8fa55343..3268ead9 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -43,6 +43,10 @@ org.springframework.boot spring-boot-starter-web + + org.springframework.boot + spring-boot-starter-webflux + com.fasterxml.jackson.module jackson-module-kotlin diff --git a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/ConsoleLogService.kt b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/ConsoleLogService.kt deleted file mode 100644 index 8cb38571..00000000 --- a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/ConsoleLogService.kt +++ /dev/null @@ -1,8 +0,0 @@ -package ch.sbb.backend.domain.logging - -class ConsoleLogService : LogService { - - override fun logs(logEntries: List) { - logEntries.forEach { println(it) } - } -} diff --git a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/LogEntry.kt b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/LogEntry.kt index c4c4fa38..e663543e 100644 --- a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/LogEntry.kt +++ b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/LogEntry.kt @@ -9,7 +9,14 @@ data class LogEntry( private val level: LogLevel, private val metadata: Map? = emptyMap() ) { - override fun toString(): String { - return "${time}\t${level}\t${source}\t${message} ${metadata}" + fun toSplunkRequest(): SplunkRequest { + val fields = metadata?.toMutableMap() ?: mutableMapOf() + fields["level"] = level.name + return SplunkRequest( + event = message, + fields = fields, + source = source, + time = time, + ) } } diff --git a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/MultiTenantLogService.kt b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/MultiTenantLogService.kt index 619d8edb..5ec9b437 100644 --- a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/MultiTenantLogService.kt +++ b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/MultiTenantLogService.kt @@ -7,21 +7,17 @@ import ch.sbb.backend.domain.tenancy.ConfigTenantService import ch.sbb.backend.domain.tenancy.TenantId import ch.sbb.backend.infrastructure.configuration.LogDestination import org.springframework.stereotype.Service -import java.util.* @Service -class MultitenantLogService(private val tenantService: ConfigTenantService) { - private val logServices: EnumMap = - EnumMap(LogDestination::class.java) +class MultitenantLogService( + private val tenantService: ConfigTenantService, + private val splunkLogService: SplunkLogService +) { fun logs(logs: List) { - getLogService(TenantContext.current().tenantId)?.logs(logs.map { + getLogService(TenantContext.current().tenantId).logs(logs.map { LogEntry( - it.time, - it.source, - it.message, - level(it.level), - it.metadata + it.time, it.source, it.message, level(it.level), it.metadata ) }) } @@ -37,21 +33,10 @@ class MultitenantLogService(private val tenantService: ConfigTenantService) { } } - private fun getLogService(tenantId: TenantId): LogService? { + private fun getLogService(tenantId: TenantId): LogService { val logDestination = tenantService.getById(tenantId).logDestination - if (logServices[logDestination] != null) { - return logServices[logDestination] - } - val logService: LogService = createLogService(logDestination) - logServices[logDestination] = logService - return logService - } - - private fun createLogService(logDestination: LogDestination?): LogService { return when (logDestination) { - LogDestination.CONSOLE -> ConsoleLogService() - LogDestination.SPLUNK -> SplunkLogService() - else -> ConsoleLogService() + LogDestination.SPLUNK -> splunkLogService } } } diff --git a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/SplunkLogService.kt b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/SplunkLogService.kt index 620059a3..b1ed99cc 100644 --- a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/SplunkLogService.kt +++ b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/SplunkLogService.kt @@ -1,13 +1,31 @@ package ch.sbb.backend.domain.logging import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value +import org.springframework.http.HttpStatus +import org.springframework.stereotype.Service +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.WebClientResponseException +import org.springframework.web.server.ResponseStatusException -/** example of a log service implementation without the expected behaviour */ -class SplunkLogService: LogService { +@Service +class SplunkLogService( + @Value("\${splunk.url}") private val url: String, + @Value("\${splunk.token}") private val token: String +) : LogService { private val log = LoggerFactory.getLogger(SplunkLogService::class.java) + private val webClient: WebClient = WebClient.create(url) - override fun logs( logEntries: List) { - logEntries.forEach { log.info("SPLUNK: $it") } - // todo: instead of logging to console, log to splunk + override fun logs(logEntries: List) { + webClient.post() + .headers { it["Authorization"] = "Splunk $token" } + .bodyValue(logEntries.map { it.toSplunkRequest() }) + .retrieve() + .bodyToMono(Object::class.java) + .doOnError(WebClientResponseException::class.java) { + log.error("Error sending logs to Splunk status=${it.statusCode} body=${it.responseBodyAsString}") + throw ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR) + } + .block() } } diff --git a/backend/src/main/kotlin/ch/sbb/backend/domain/logging/SplunkRequest.kt b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/SplunkRequest.kt new file mode 100644 index 00000000..dfc0332c --- /dev/null +++ b/backend/src/main/kotlin/ch/sbb/backend/domain/logging/SplunkRequest.kt @@ -0,0 +1,17 @@ +package ch.sbb.backend.domain.logging + +import com.fasterxml.jackson.annotation.JsonInclude +import java.time.OffsetDateTime + +data class SplunkRequest( + val event: String, + val fields: Map, + @JsonInclude(JsonInclude.Include.NON_NULL) + val host: String? = null, + @JsonInclude(JsonInclude.Include.NON_NULL) + val index: String? = null, + val source: String, + val time: OffsetDateTime, + @JsonInclude(JsonInclude.Include.NON_NULL) + val sourcetype: String? = null +) diff --git a/backend/src/main/kotlin/ch/sbb/backend/domain/tenancy/ConfigTenantService.kt b/backend/src/main/kotlin/ch/sbb/backend/domain/tenancy/ConfigTenantService.kt index e8289344..15f1a061 100644 --- a/backend/src/main/kotlin/ch/sbb/backend/domain/tenancy/ConfigTenantService.kt +++ b/backend/src/main/kotlin/ch/sbb/backend/domain/tenancy/ConfigTenantService.kt @@ -26,7 +26,7 @@ class ConfigTenantService(private val tenantConfig: TenantConfig) : TenantServic } override fun getById(tenantId: TenantId): Tenant { - return tenantConfig.tenants.stream().filter { t -> tenantId == TenantId(t.id!!) } + return tenantConfig.tenants.stream().filter { t -> tenantId == TenantId(t.id) } .findAny() .orElseThrow { IllegalArgumentException("unknown tenant") } } diff --git a/backend/src/main/kotlin/ch/sbb/backend/infrastructure/configuration/LogDestination.kt b/backend/src/main/kotlin/ch/sbb/backend/infrastructure/configuration/LogDestination.kt index c67ba0c7..0fbf5119 100644 --- a/backend/src/main/kotlin/ch/sbb/backend/infrastructure/configuration/LogDestination.kt +++ b/backend/src/main/kotlin/ch/sbb/backend/infrastructure/configuration/LogDestination.kt @@ -1,6 +1,5 @@ package ch.sbb.backend.infrastructure.configuration enum class LogDestination { - CONSOLE, SPLUNK } diff --git a/backend/src/main/kotlin/ch/sbb/backend/infrastructure/configuration/Tenant.kt b/backend/src/main/kotlin/ch/sbb/backend/infrastructure/configuration/Tenant.kt index 4da1a58e..d6cdb54a 100644 --- a/backend/src/main/kotlin/ch/sbb/backend/infrastructure/configuration/Tenant.kt +++ b/backend/src/main/kotlin/ch/sbb/backend/infrastructure/configuration/Tenant.kt @@ -1,9 +1,9 @@ package ch.sbb.backend.infrastructure.configuration data class Tenant( - var name: String? = null, - var id: String? = null, - var jwkSetUri: String? = null, - var issuerUri: String? = null, - var logDestination: LogDestination? = null + var name: String, + var id: String, + var jwkSetUri: String, + var issuerUri: String, + var logDestination: LogDestination ) diff --git a/backend/src/main/resources/application.yaml b/backend/src/main/resources/application.yaml index eee272ac..760a80a4 100644 --- a/backend/src/main/resources/application.yaml +++ b/backend/src/main/resources/application.yaml @@ -30,9 +30,13 @@ auth: id: 2cda5d11-f0ac-46b3-967d-af1b2e1bd01a issuer-uri: https://login.microsoftonline.com/2cda5d11-f0ac-46b3-967d-af1b2e1bd01a/v2.0 jwk-set-uri: https://login.microsoftonline.com/2cda5d11-f0ac-46b3-967d-af1b2e1bd01a/discovery/v2.0/keys - log-destination: console + log-destination: splunk - name: sob id: d653d01f-17a4-48a1-9aab-b780b61b4273 issuer-uri: https://login.microsoftonline.com/d653d01f-17a4-48a1-9aab-b780b61b4273/v2.0 jwk-set-uri: https://login.microsoftonline.com/d653d01f-17a4-48a1-9aab-b780b61b4273/discovery/v2.0/keys log-destination: splunk + +splunk: + url: ${SPLUNK_HEC_URL} + token: ${SPLUNK_HEC_TOKEN} From 475a4a41983b20d718f026445924bf236b1a41aa Mon Sep 17 00:00:00 2001 From: mghilardelli Date: Fri, 8 Nov 2024 08:52:05 +0100 Subject: [PATCH 2/2] fix: tests --- .../application/rest/LoggingControllerTest.kt | 63 +++++++++++-------- .../logging/MultitenantLogServiceTest.kt | 37 ++++------- .../configuration/TenantConfigTest.kt | 12 ++-- .../src/test/resources/application-test.yaml | 7 --- backend/src/test/resources/application.yaml | 31 +++++++++ 5 files changed, 89 insertions(+), 61 deletions(-) delete mode 100644 backend/src/test/resources/application-test.yaml create mode 100644 backend/src/test/resources/application.yaml diff --git a/backend/src/test/kotlin/ch/sbb/backend/application/rest/LoggingControllerTest.kt b/backend/src/test/kotlin/ch/sbb/backend/application/rest/LoggingControllerTest.kt index 4b8e0a53..c3358eac 100644 --- a/backend/src/test/kotlin/ch/sbb/backend/application/rest/LoggingControllerTest.kt +++ b/backend/src/test/kotlin/ch/sbb/backend/application/rest/LoggingControllerTest.kt @@ -1,20 +1,21 @@ package ch.sbb.backend.application.rest -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.BeforeEach +import ch.sbb.backend.domain.logging.LogEntry +import ch.sbb.backend.domain.logging.LogLevel +import ch.sbb.backend.domain.logging.SplunkLogService import org.junit.jupiter.api.Test +import org.mockito.Mockito.verify import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.MockBean import org.springframework.http.MediaType import org.springframework.security.core.authority.SimpleGrantedAuthority import org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.jwt import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status -import java.io.ByteArrayOutputStream -import java.io.PrintStream +import java.time.OffsetDateTime @SpringBootTest @AutoConfigureMockMvc @@ -23,18 +24,8 @@ class LoggingControllerTest { @Autowired private lateinit var mockMvc: MockMvc - private val originalOut = System.out - private val outputStreamCaptor = ByteArrayOutputStream() - - @BeforeEach - fun setUp() { - System.setOut(PrintStream(outputStreamCaptor)) - } - - @AfterEach - fun tearDown() { - System.setOut(originalOut) - } + @MockBean + private lateinit var splunkLogService: SplunkLogService @Test fun `should log messages with seconds since epoch timestamp`() { @@ -57,7 +48,7 @@ class LoggingControllerTest { jwt.claims { claims -> claims.put( "tid", - "2cda5d11-f0ac-46b3-967d-af1b2e1bd01a" + "3409e798-d567-49b1-9bae-f0be66427c54" ) } } @@ -68,8 +59,16 @@ class LoggingControllerTest { ) .andExpect(status().isOk) - val output = outputStreamCaptor.toString() - assertTrue(output.contains("2024-10-05T18:58:19.452+02:00\tINFO\titest\tmy message {}")) + val expectedLogs = listOf( + LogEntry( + OffsetDateTime.parse("2024-10-05T18:58:19.452+02:00"), + "itest", + "my message", + LogLevel.INFO + ) + ) + + verify(splunkLogService).logs(expectedLogs) } @Test @@ -103,7 +102,7 @@ class LoggingControllerTest { jwt.claims { claims -> claims.put( "tid", - "d653d01f-17a4-48a1-9aab-b780b61b4273" + "3409e798-d567-49b1-9bae-f0be66427c54" ) } } @@ -114,9 +113,23 @@ class LoggingControllerTest { ) .andExpect(status().isOk) - val output = outputStreamCaptor.toString() - assertTrue(output.lines()[0].contains("SPLUNK: 2024-10-01T14:34:56.789+02:00\tERROR\titest\tmy message {key1=value1, key2=value2}")) - assertTrue(output.lines()[1].contains("SPLUNK: 2024-10-01T14:36:12.546+02:00\tWARNING\titest\tmy warning {}")) + val expectedLogs = listOf( + LogEntry( + OffsetDateTime.parse("2024-10-01T14:34:56.789+02:00"), + "itest", + "my message", + LogLevel.ERROR, + mapOf("key1" to "value1", "key2" to "value2") + ), + LogEntry( + OffsetDateTime.parse("2024-10-01T14:36:12.546+02:00"), + "itest", + "my warning", + LogLevel.WARNING + ) + ) + + verify(splunkLogService).logs(expectedLogs) } @Test @@ -139,7 +152,7 @@ class LoggingControllerTest { jwt.claims { claims -> claims.put( "tid", - "2cda5d11-f0ac-46b3-967d-af1b2e1bd01a" + "3409e798-d567-49b1-9bae-f0be66427c54" ) } } diff --git a/backend/src/test/kotlin/ch/sbb/backend/domain/logging/MultitenantLogServiceTest.kt b/backend/src/test/kotlin/ch/sbb/backend/domain/logging/MultitenantLogServiceTest.kt index da1fd859..324bfe67 100644 --- a/backend/src/test/kotlin/ch/sbb/backend/domain/logging/MultitenantLogServiceTest.kt +++ b/backend/src/test/kotlin/ch/sbb/backend/domain/logging/MultitenantLogServiceTest.kt @@ -6,39 +6,31 @@ import ch.sbb.backend.domain.tenancy.ConfigTenantService import ch.sbb.backend.domain.tenancy.TenantId import ch.sbb.backend.infrastructure.configuration.LogDestination import ch.sbb.backend.infrastructure.configuration.Tenant -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.mockito.Mockito.mock -import org.mockito.Mockito.`when` +import org.mockito.Mockito.* import org.springframework.security.core.Authentication import org.springframework.security.core.context.SecurityContext import org.springframework.security.core.context.SecurityContextHolder import org.springframework.security.oauth2.jwt.Jwt -import java.io.ByteArrayOutputStream -import java.io.PrintStream import java.time.Instant import java.time.OffsetDateTime import java.util.* class MultitenantLogServiceTest { - private lateinit var tenantService: ConfigTenantService private lateinit var sut: MultitenantLogService - - private val originalOut = System.out - private val outputStreamCaptor = ByteArrayOutputStream() + private lateinit var tenantService: ConfigTenantService + private lateinit var splunkLogService: SplunkLogService private val tid = UUID.randomUUID().toString() @BeforeEach fun setUp() { tenantService = mock(ConfigTenantService::class.java) - sut = MultitenantLogService(tenantService) + splunkLogService = mock(SplunkLogService::class.java) + sut = MultitenantLogService(tenantService, splunkLogService) val securityContext: SecurityContext = mock(SecurityContext::class.java) - System.setOut(PrintStream(outputStreamCaptor)) - val jwt = Jwt( "token", Instant.now(), @@ -53,26 +45,21 @@ class MultitenantLogServiceTest { } @Test - fun `should log messages to console`() { + fun `should log messages to splunk`() { val tenantId = TenantId(tid) - val tenantConfig = Tenant().apply { - logDestination = LogDestination.CONSOLE - } + val tenantConfig = Tenant("test", "10", "", "", LogDestination.SPLUNK) `when`(tenantService.getById(tenantId)).thenReturn(tenantConfig) - val timestamp = OffsetDateTime.now() + val timestamp = OffsetDateTime.now() val logEntries = listOf( LogEntryRequest(timestamp, "source", "message", LogLevelRequest.INFO) ) sut.logs(logEntries) - val output = outputStreamCaptor.toString() - assertTrue(output.contains("${timestamp.toString()}\tINFO\tsource\tmessage {}\n")) - } - - @AfterEach - fun tearDown() { - System.setOut(originalOut) + val expectedLogs = listOf( + LogEntry(timestamp, "source", "message", LogLevel.INFO) + ) + verify(splunkLogService, times(1)).logs(expectedLogs) } } diff --git a/backend/src/test/kotlin/ch/sbb/backend/infrastructure/configuration/TenantConfigTest.kt b/backend/src/test/kotlin/ch/sbb/backend/infrastructure/configuration/TenantConfigTest.kt index 0660452d..8b50c39f 100644 --- a/backend/src/test/kotlin/ch/sbb/backend/infrastructure/configuration/TenantConfigTest.kt +++ b/backend/src/test/kotlin/ch/sbb/backend/infrastructure/configuration/TenantConfigTest.kt @@ -4,10 +4,8 @@ import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.test.context.ActiveProfiles @SpringBootTest -@ActiveProfiles("test") class TenantConfigTest { @Autowired @@ -21,7 +19,13 @@ class TenantConfigTest { assertEquals(tenants.size, 1) assertEquals(tenants[0].name, "test") assertEquals(tenants[0].id, "3409e798-d567-49b1-9bae-f0be66427c54") - assertEquals(tenants[0].issuerUri, "https://login.microsoftonline.com/3409e798-d567-49b1-9bae-f0be66427c54/v2.0") - assertEquals(tenants[0].jwkSetUri, "https://login.microsoftonline.com/3409e798-d567-49b1-9bae-f0be66427c54/discovery/v2.0/keys") + assertEquals( + tenants[0].issuerUri, + "https://login.microsoftonline.com/3409e798-d567-49b1-9bae-f0be66427c54/v2.0" + ) + assertEquals( + tenants[0].jwkSetUri, + "https://login.microsoftonline.com/3409e798-d567-49b1-9bae-f0be66427c54/discovery/v2.0/keys" + ) } } diff --git a/backend/src/test/resources/application-test.yaml b/backend/src/test/resources/application-test.yaml deleted file mode 100644 index 2d240063..00000000 --- a/backend/src/test/resources/application-test.yaml +++ /dev/null @@ -1,7 +0,0 @@ -auth: - tenants: - - name: test - id: 3409e798-d567-49b1-9bae-f0be66427c54 - issuer-uri: https://login.microsoftonline.com/3409e798-d567-49b1-9bae-f0be66427c54/v2.0 - jwk-set-uri: https://login.microsoftonline.com/3409e798-d567-49b1-9bae-f0be66427c54/discovery/v2.0/keys - log-destination: console diff --git a/backend/src/test/resources/application.yaml b/backend/src/test/resources/application.yaml new file mode 100644 index 00000000..35d373e7 --- /dev/null +++ b/backend/src/test/resources/application.yaml @@ -0,0 +1,31 @@ +info: + app: + version: test + +spring: + security: + oauth2: + authorizationUrl: test + jackson: + mapper: + accept-case-insensitive-enums: true + time-zone: CET + +springdoc: + swagger-ui: + oauth: + clientId: test + +auth: + audience: + service-name: test + tenants: + - name: test + id: 3409e798-d567-49b1-9bae-f0be66427c54 + issuer-uri: https://login.microsoftonline.com/3409e798-d567-49b1-9bae-f0be66427c54/v2.0 + jwk-set-uri: https://login.microsoftonline.com/3409e798-d567-49b1-9bae-f0be66427c54/discovery/v2.0/keys + log-destination: splunk + +splunk: + url: "url" + token: "token"