From e088b8911a9de658a2ee596ea5d347d02ab811ef Mon Sep 17 00:00:00 2001 From: August Kilponen Date: Tue, 2 Apr 2024 14:20:04 +0300 Subject: [PATCH] =?UTF-8?q?Lis=C3=A4=C3=A4=20lokitusta=20hakemusten=20hake?= =?UTF-8?q?miseen=20YTL-ajossa.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration/ytl/YtlIntegration.scala | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/scala/fi/vm/sade/hakurekisteri/integration/ytl/YtlIntegration.scala b/src/main/scala/fi/vm/sade/hakurekisteri/integration/ytl/YtlIntegration.scala index e0dde1384..741c80d00 100644 --- a/src/main/scala/fi/vm/sade/hakurekisteri/integration/ytl/YtlIntegration.scala +++ b/src/main/scala/fi/vm/sade/hakurekisteri/integration/ytl/YtlIntegration.scala @@ -112,9 +112,19 @@ class YtlIntegration( .map(_.flatten) } - hakuOids.grouped(10).foldLeft(Future.successful(Set.empty[HetuPersonOid])) { - case (result, chunk) => result.flatMap(rs => fetchChunk(chunk).map(rs ++ _)) - } + val hakuOidsChunkSize = 10 + hakuOids.zipWithIndex + .grouped(hakuOidsChunkSize) + .foldLeft(Future.successful(Set.empty[HetuPersonOid])) { + case (result, chunkWithIndex) => { + val chunk = chunkWithIndex.map(_._1) + val firstIndex = chunkWithIndex.map(_._2).head + logger.info( + s"Fetching hakuOid chunk. First hakuOid is ${firstIndex}/${hakuOids.size} (Chunk size is ${hakuOidsChunkSize} and hakuOids are ${chunk})" + ) + result.flatMap(rs => fetchChunk(chunk).map(rs ++ _)) + } + } } logger.info(s"Fetching in chunks, activeKKHakuOids: ${activeKKHakuOids.get()}")