Making max payload size configurable (#1154)

server/parsing/grouped-results-parser/src/main/java/projektor/parser/grouped/GroupedResultsParser.java

@@ -9,16 +9,20 @@
 import java.io.IOException;
 
 public class GroupedResultsParser {
-    private static final int MAX_PAYLOAD_STRING_LENGTH = 50_000_000; // default is 20_000_000
+    public static final int DEFAULT_MAX_PAYLOAD_SIZE = 50_000_000; // Jackson default is 20_000_000
 
     private final ObjectMapper mapper;
 
     public GroupedResultsParser() {
+        this(DEFAULT_MAX_PAYLOAD_SIZE);
+    }
+
+    public GroupedResultsParser(Integer maxPayloadSize) {
         this.mapper = new ObjectMapper()
                 .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
         this.mapper.getFactory()
-                .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(MAX_PAYLOAD_STRING_LENGTH).build());
+                .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxPayloadSize).build());
     }
 
     public GroupedResults parseGroupedResults(String groupedResultsBlob) throws IOException {

server/parsing/grouped-results-parser/src/test/groovy/projektor/parser/grouped/GroupedResultsParserSpec.groovy

@@ -7,16 +7,17 @@ import projektor.parser.grouped.model.GroupedTestSuites
 import projektor.parser.grouped.model.PerformanceResult
 import projektor.parser.grouped.model.ResultsMetadata
 import spock.lang.Specification
-import spock.lang.Subject
 
 class GroupedResultsParserSpec extends Specification {
-    @Subject
-    GroupedResultsParser groupedResultsParser = new GroupedResultsParser()
+    private int defaultMaxPayloadSize = 20_000_000
+    private int largeMaxPayloadSize = 50_000_000
 
     private ObjectMapper mapper = new ObjectMapper()
 
     def "should deserialize grouped results that were serialized"() {
         given:
+        GroupedResultsParser groupedResultsParser = new GroupedResultsParser(defaultMaxPayloadSize)
+
         List<GroupedTestSuites> groupedTestSuites = (1..3).collect { idx ->
             new GroupedTestSuites(
                     groupName: "MyGroup${idx}",
@@ -56,6 +57,8 @@ class GroupedResultsParserSpec extends Specification {
 
     def "should serialize grouped results"() {
         given:
+        GroupedResultsParser groupedResultsParser = new GroupedResultsParser(defaultMaxPayloadSize)
+
         List<GroupedTestSuites> groupedTestSuites = (1..2).collect { idx ->
             new GroupedTestSuites(
                     groupName: "MyGroup${idx}",
@@ -83,6 +86,8 @@ class GroupedResultsParserSpec extends Specification {
 
     def "should parse grouped results with all metadata"() {
         given:
+        GroupedResultsParser groupedResultsParser = new GroupedResultsParser(defaultMaxPayloadSize)
+
         List<GroupedTestSuites> groupedTestSuites = (1..2).collect { idx ->
             new GroupedTestSuites(
                     groupName: "MyGroup${idx}",
@@ -120,6 +125,8 @@ class GroupedResultsParserSpec extends Specification {
 
     def "should deserialize results with only performance results"() {
         given:
+        GroupedResultsParser groupedResultsParser = new GroupedResultsParser(defaultMaxPayloadSize)
+
         GroupedResults groupedResults = new GroupedResults()
         groupedResults.metadata = new ResultsMetadata(
                 git: new GitMetadata(repoName: "my-repo", branchName: "main", isMainBranch: true)
@@ -138,6 +145,8 @@ class GroupedResultsParserSpec extends Specification {
     }
 
     def "should not fail on really large payloads"() {
+        GroupedResultsParser groupedResultsParser = new GroupedResultsParser(largeMaxPayloadSize)
+
         String largeBlob = (1..5_000_000).collect { i -> i }.join()
 
         GroupedTestSuites groupedTestSuite = new GroupedTestSuites(

server/server-app/src/main/kotlin/projektor/AppModule.kt

@@ -44,6 +44,7 @@ import projektor.parser.performance.PerformanceResultsParser
 import projektor.performance.PerformanceResultsDatabaseRepository
 import projektor.performance.PerformanceResultsRepository
 import projektor.performance.PerformanceResultsService
+import projektor.processing.ProcessingConfig
 import projektor.quality.CodeQualityReportDatabaseRepository
 import projektor.quality.CodeQualityReportRepository
 import projektor.repository.coverage.RepositoryCoverageDatabaseRepository
@@ -78,6 +79,7 @@ fun createAppModule(
     metricRegistry: MeterRegistry,
     messageConfig: MessageConfig,
     notificationConfig: NotificationConfig,
+    processingConfig: ProcessingConfig,
     gitHubCommentService: GitHubCommentService?,
     attachmentService: AttachmentService?
 ) = module {
@@ -111,7 +113,7 @@ fun createAppModule(
 
     single { AppendTestResultsService(get(), get()) }
     single { CoverageService(get(), get(), get(), get()) }
-    single { GroupedResultsParser() }
+    single { GroupedResultsParser(processingConfig.maxPayloadSize) }
     single { PerformanceResultsParser() }
     single { GroupedResultsConverter(get(), get(), get()) }
     single { GroupedTestResultsService(get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }

server/server-app/src/main/kotlin/projektor/Application.kt

@@ -58,6 +58,7 @@ import projektor.notification.github.comment.GitHubCommentClient
 import projektor.notification.github.comment.GitHubCommentService
 import projektor.organization.coverage.OrganizationCoverageService
 import projektor.performance.PerformanceResultsService
+import projektor.processing.ProcessingConfig
 import projektor.quality.CodeQualityReportRepository
 import projektor.repository.coverage.RepositoryCoverageService
 import projektor.repository.performance.RepositoryPerformanceService
@@ -112,6 +113,8 @@ fun Application.main(meterRegistry: MeterRegistry? = null) {
 
     val versionControlConfig = VersionControlConfig.createVersionControlConfig(applicationConfig)
 
+    val processingConfig = ProcessingConfig.createProcessingConfig(applicationConfig)
+
     val notificationConfig = NotificationConfig.createNotificationConfig(applicationConfig)
     val gitHubCommentService = conditionallyCreateGitHubCommentService(applicationConfig)
 
@@ -126,6 +129,7 @@ fun Application.main(meterRegistry: MeterRegistry? = null) {
         metricRegistry = metricRegistry,
         messageConfig = messageConfig,
         notificationConfig = notificationConfig,
+        processingConfig = processingConfig,
         gitHubCommentService = gitHubCommentService,
         attachmentService = attachmentService
     )

server/server-app/src/main/kotlin/projektor/processing/ProcessingConfig.kt

@@ -0,0 +1,14 @@
+package projektor.processing
+
+import io.ktor.server.config.*
+import projektor.parser.grouped.GroupedResultsParser
+
+data class ProcessingConfig(val maxPayloadSize: Int) {
+    companion object {
+        fun createProcessingConfig(applicationConfig: ApplicationConfig): ProcessingConfig {
+            val maxPayloadSize: Int = applicationConfig.propertyOrNull("ktor.processing.maxPayloadSize")?.toString()?.toInt() ?: GroupedResultsParser.DEFAULT_MAX_PAYLOAD_SIZE
+
+            return ProcessingConfig(maxPayloadSize)
+        }
+    }
+}

server/server-app/src/main/resources/application.conf

@@ -70,4 +70,8 @@ ktor {
             privateKey = ${?GITHUB_PRIVATE_KEY}
         }
     }
+
+    processing {
+        maxPayloadSize = ${?MAX_PAYLOAD_SIZE}
+    }
 }

server/server-app/src/test/kotlin/projektor/DatabaseRepositoryTestCase.kt

@@ -39,6 +39,8 @@ import projektor.metrics.InfluxMetricsConfig
 import projektor.metrics.MetricsService
 import projektor.metrics.createRegistry
 import projektor.notification.NotificationConfig
+import projektor.parser.grouped.GroupedResultsParser
+import projektor.processing.ProcessingConfig
 import java.util.TimeZone
 
 open class DatabaseRepositoryTestCase : KoinTest {
@@ -120,6 +122,7 @@ open class DatabaseRepositoryTestCase : KoinTest {
                     createRegistry(metricsConfig),
                     MessageConfig(listOf()),
                     NotificationConfig(null),
+                    ProcessingConfig(GroupedResultsParser.DEFAULT_MAX_PAYLOAD_SIZE),
                     null,
                     null
                 )