Skip to content

Commit 4499e60

Browse files
jordimassaguerplajordimassaguerpla
committed
Merge branch 'develop' of https://github.com/Project-MONAI/monai-deploy-workflow-manager into jordi_develop
* 'develop' of https://github.com/Project-MONAI/monai-deploy-workflow-manager: AC-1243 Added configurable resource constraints on argo message sending pod containers added mongo migrations
2 parents c796429 + d36b557 commit 4499e60

File tree

58 files changed

+5482
-299
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+5482
-299
lines changed

‎doc/dependency_decisions.yml

Lines changed: 105 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
:why: Apache-2.0 (https://github.com/MichaCo/DnsClient.NET/raw/dev/LICENSE)
7070
:versions:
7171
- 1.6.1
72+
- 1.4.0
7273
:when: 2022-10-14 23:36:42.746807998 Z
7374
- - :approve
7475
- Docker.DotNet
@@ -235,14 +236,8 @@
235236
:why: Apache-2.0 (https://raw.githubusercontent.com/aspnet/AspNetCore/2.0.0/LICENSE.txt)
236237
:versions:
237238
- 2.1.0
238-
:when: 2022-10-14 23:36:52.062635438 Z
239-
- - :approve
240-
- Microsoft.AspNetCore.Http.Abstractions
241-
- :who: mocsharp
242-
:why: Apache-2.0 (https://raw.githubusercontent.com/aspnet/AspNetCore/2.0.0/LICENSE.txt)
243-
:versions:
244239
- 2.2.0
245-
:when: 2022-10-14 23:36:52.442347272 Z
240+
:when: 2022-10-14 23:36:52.062635438 Z
246241
- - :approve
247242
- Microsoft.AspNetCore.Http.Extensions
248243
- :who: mocsharp
@@ -405,6 +400,7 @@
405400
:why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
406401
:versions:
407402
- 3.0.0
403+
- 2.2.0
408404
:when: 2022-10-14 23:37:01.366955631 Z
409405
- - :approve
410406
- Microsoft.Extensions.Configuration.Binder
@@ -455,6 +451,7 @@
455451
:versions:
456452
- 6.0.0
457453
- 6.0.1
454+
- 2.2.0
458455
:when: 2022-10-14 23:37:03.975202345 Z
459456
- - :approve
460457
- Microsoft.Extensions.DependencyInjection.Abstractions
@@ -563,13 +560,15 @@
563560
:why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
564561
:versions:
565562
- 6.0.2
563+
- 2.0.0
566564
:when: 2022-10-14 23:37:09.821156462 Z
567565
- - :approve
568566
- Microsoft.Extensions.Logging.Configuration
569567
- :who: mocsharp
570568
:why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
571569
:versions:
572570
- 3.0.0
571+
- 2.2.0
573572
:when: 2022-10-14 23:37:10.225726023 Z
574573
- - :approve
575574
- Microsoft.Extensions.Logging.Configuration
@@ -584,13 +583,16 @@
584583
:why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
585584
:versions:
586585
- 6.0.0
586+
- 2.2.0
587587
:when: 2022-10-14 23:37:11.015778545 Z
588588
- - :approve
589589
- Microsoft.Extensions.Logging.Debug
590590
- :who: mocsharp
591591
:why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
592592
:versions:
593593
- 6.0.0
594+
- 2.0.0
595+
- 2.2.0
594596
:when: 2022-10-14 23:37:11.396975779 Z
595597
- - :approve
596598
- Microsoft.Extensions.Logging.EventLog
@@ -619,13 +621,16 @@
619621
:why: Apache-2.0 (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
620622
:versions:
621623
- 6.0.0
624+
- 2.0.0
622625
:when: 2022-10-14 23:37:12.901607973 Z
623626
- - :approve
624627
- Microsoft.Extensions.Options.ConfigurationExtensions
625628
- :who: mocsharp
626629
:why: Apache-2.0 (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
627630
:versions:
628631
- 3.0.0
632+
- 2.0.0
633+
- 2.2.0
629634
:when: 2022-10-14 23:37:13.281680374 Z
630635
- - :approve
631636
- Microsoft.Extensions.Options.ConfigurationExtensions
@@ -723,6 +728,7 @@
723728
:why: MICROSOFT .NET LIBRARY License (http://go.microsoft.com/fwlink/?LinkId=329770)
724729
:versions:
725730
- 1.1.0
731+
- 2.1.2
726732
:when: 2022-10-14 23:37:18.412134581 Z
727733
- - :approve
728734
- Microsoft.NETCore.Platforms
@@ -887,14 +893,10 @@
887893
:why: Apache-2.0 (https://github.com/serilog/serilog/raw/dev/LICENSE)
888894
:versions:
889895
- 2.10.0
890-
:when: 2022-10-14 23:37:30.926882753 Z
891-
- - :approve
892-
- Serilog
893-
- :who: mocsharp
894-
:why: Apache-2.0 (https://github.com/serilog/serilog/raw/dev/LICENSE)
895-
:versions:
896896
- 2.12.0
897-
:when: 2022-10-14 23:37:31.312837906 Z
897+
- 2.8.0
898+
:when: 2022-10-14 23:37:30.926882753 Z
899+
898900
- - :approve
899901
- Serilog.AspNetCore
900902
- :who: mocsharp
@@ -964,6 +966,21 @@
964966
:why: Apache-2.0 (https://github.com/serilog/serilog/raw/dev/LICENSE)
965967
:versions:
966968
- 5.0.0
969+
- 3.2.0
970+
:when: 2022-10-14 23:37:35.402868961 Z
971+
- - :approve
972+
- Serilog.Sinks.RollingFile
973+
- :who: neildsouth
974+
:why: Apache-2.0 (https://github.com/serilog/serilog/raw/dev/LICENSE)
975+
:versions:
976+
- 3.3.0
977+
:when: 2022-10-14 23:37:35.402868961 Z
978+
- - :approve
979+
- Serilog.Sinks.Async
980+
- :who: neildsouth
981+
:why: Apache-2.0 (https://github.com/serilog/serilog/raw/dev/LICENSE)
982+
:versions:
983+
- 1.1.0
967984
:when: 2022-10-14 23:37:35.402868961 Z
968985
- - :approve
969986
- Serilog.Sinks.Http
@@ -978,6 +995,7 @@
978995
:why: MIT (https://github.com/adamhathcock/sharpcompress/raw/master/LICENSE.txt)
979996
:versions:
980997
- 0.30.1
998+
- 0.23.0
981999
:when: 2022-10-14 23:37:36.289199068 Z
9821000
- - :approve
9831001
- Snappier
@@ -1125,6 +1143,7 @@
11251143
:why: MICROSOFT .NET LIBRARY License (http://go.microsoft.com/fwlink/?LinkId=329770)
11261144
:versions:
11271145
- 4.3.0
1146+
- 4.0.12
11281147
:when: 2022-10-14 23:37:44.795018964 Z
11291148
- - :approve
11301149
- System.ComponentModel.Annotations
@@ -1209,6 +1228,7 @@
12091228
:why: MICROSOFT .NET LIBRARY License (http://go.microsoft.com/fwlink/?LinkId=329770)
12101229
:versions:
12111230
- 4.3.0
1231+
- 4.1.0
12121232
:when: 2022-10-14 23:37:49.133631683 Z
12131233
- - :approve
12141234
- System.Dynamic.Runtime
@@ -1886,13 +1906,15 @@
18861906
:why: MICROSOFT .NET LIBRARY License (http://go.microsoft.com/fwlink/?LinkId=329770)
18871907
:versions:
18881908
- 4.3.0
1909+
- 4.0.1
18891910
:when: 2022-10-14 23:38:29.841118580 Z
18901911
- - :approve
18911912
- System.ValueTuple
18921913
- :who: mocsharp
18931914
:why: MIT (https://github.com/dotnet/corefx/raw/master/LICENSE.TXT)
18941915
:versions:
18951916
- 4.4.0
1917+
- 4.5.0
18961918
:when: 2022-10-14 23:38:30.241703462 Z
18971919
- - :approve
18981920
- System.Xml.ReaderWriter
@@ -2205,13 +2227,6 @@
22052227
:versions:
22062228
- 2.4.5
22072229
:when: 2022-10-14 23:38:48.802854287 Z
2208-
- - :approve
2209-
- NLog
2210-
- :who: mocsharp
2211-
:why: BSD 3-Clause License (https://github.com/NLog/NLog/raw/dev/LICENSE.txt)
2212-
:versions:
2213-
- 5.1.0
2214-
:when: 2022-10-12 03:14:06.538744982 Z
22152230
- - :approve
22162231
- NLog.Extensions.Logging
22172232
- :who: mocsharp
@@ -2304,26 +2319,21 @@
23042319
:versions:
23052320
- 2.18.0
23062321
:when: 2022-11-02 21:44:41.801284907 Z
2307-
- - :approve
2308-
- MongoDB.Driver
2309-
- :who: mocsharp
2310-
:why: Apache-2.0 (https://github.com/mongodb/mongo-csharp-driver/raw/master/License.txt)
2311-
:versions:
2312-
- 2.18.0
2313-
:when: 2022-11-02 21:45:01.214220067 Z
23142322
- - :approve
23152323
- MongoDB.Driver.Core
23162324
- :who: mocsharp
23172325
:why: Apache-2.0 (https://github.com/mongodb/mongo-csharp-driver/raw/master/License.txt)
23182326
:versions:
23192327
- 2.18.0
2328+
- 2.13.1
23202329
:when: 2022-11-02 21:45:23.777282609 Z
23212330
- - :approve
23222331
- MongoDB.Libmongocrypt
23232332
- :who: mocsharp
23242333
:why: Apache-2.0 (https://github.com/mongodb/mongo-csharp-driver/raw/master/License.txt)
23252334
:versions:
23262335
- 1.6.0
2336+
- 1.2.2
23272337
:when: 2022-11-02 21:45:54.431951720 Z
23282338
- - :approve
23292339
- Monai.Deploy.Storage
@@ -2366,3 +2376,70 @@
23662376
:versions:
23672377
- 6.0.3
23682378
:when: 2022-12-02 13:13:13.431951720 Z
2379+
- - :approve
2380+
- MongoDB.Driver
2381+
- :who: mocsharp
2382+
:why: Apache-2.0 (https://github.com/mongodb/mongo-csharp-driver/raw/master/License.txt)
2383+
:versions:
2384+
- 2.18.0
2385+
- 2.13.1
2386+
:when: 2022-11-02 21:45:01.214220067 Z
2387+
- - :approve
2388+
- NLog
2389+
- :who: mocsharp
2390+
:why: BSD 3-Clause License (https://github.com/NLog/NLog/raw/dev/LICENSE.txt)
2391+
:versions:
2392+
- 5.1.0
2393+
- 4.7.11
2394+
:when: 2022-10-12 03:14:06.538744982 Z
2395+
- - :approve
2396+
- Mongo.Migration
2397+
- :who: neildsouth
2398+
:why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
2399+
:versions:
2400+
- 3.1.4
2401+
:when: 2023-02-02 15:35:00.000000000 Z
2402+
- - :approve
2403+
- LightInject
2404+
- :who: neildsouth
2405+
:why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
2406+
:versions:
2407+
- 5.4.0
2408+
:when: 2023-02-02 15:35:00.000000000 Z
2409+
- - :approve
2410+
- Serilog.Extensions.Logging
2411+
- :who: neildsouth
2412+
:why: Apache-2.0 (https://github.com/serilog/serilog/raw/dev/LICENSE)
2413+
:versions:
2414+
- 2.0.4
2415+
:when: 2023-02-02 15:35:00.000000000 Z
2416+
- - :approve
2417+
- Serilog.Extensions.Logging.File
2418+
- :who: neildsouth
2419+
:why: Apache-2.0 (https://github.com/serilog/serilog/raw/dev/LICENSE)
2420+
:versions:
2421+
- 2.0.0
2422+
:when: 2023-02-02 15:35:00.000000000 Z
2423+
- - :approve
2424+
- Serilog.Formatting.Compact
2425+
- :who: neildsouth
2426+
:why: Apache-2.0 (https://github.com/serilog/serilog/raw/dev/LICENSE)
2427+
:versions:
2428+
- 1.0.0
2429+
:when: 2023-02-02 15:35:00.000000000 Z
2430+
2431+
- - :approve
2432+
- System.Text.Encoding.CodePages
2433+
- :who: neildsouth
2434+
:why: MIT (https://github.com/dotnet/runtime/raw/main/LICENSE.TXT)
2435+
:versions:
2436+
- 4.5.1
2437+
:when: 2023-02-02 15:35:00.000000000 Z
2438+
2439+
- - :approve
2440+
- System.Collections.NonGeneric
2441+
- :who: neildsouth
2442+
:why: MICROSOFT .NET LIBRARY License (http://go.microsoft.com/fwlink/?LinkId=329770)
2443+
:versions:
2444+
- 4.3.0
2445+
:when: 2023-02-02 15:35:00.000000000 Z

‎docs/changelog.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,7 @@
2020
Renamed the (Generated) Argo client to ArgoGeneratedClient, added a new ArgoClient using just the methods used by this codebase.
2121

2222
Enhanced the ArgoClient -> Argo_Get_WorkflowLogsAsync method to decode the json better and make the logs extracted from Argo more readable.
23+
24+
Added Mongo Migraions, to allow changes without breaking current stored entries
25+
26+
Added resource constraints to the generated ARGO templates, so these can operate within a Kubernetes cluster with a Resource Quota in the ARGO namespace.

‎src/Shared/Configuration/TaskManagerConfiguration.cs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,5 +50,29 @@ public class ArgoPluginArguments
5050
{
5151
[ConfigurationKeyName("server_url")]
5252
public string ServerUrl { get; set; } = string.Empty;
53+
54+
[ConfigurationKeyName("initContainerCpuLimit")]
55+
public string InitContainerCpuLimit { get; set; } = "1";
56+
57+
[ConfigurationKeyName("initContainerMemoryLimit")]
58+
public string InitContainerMemoryLimit { get; set; } = "500Mi";
59+
60+
[ConfigurationKeyName("waitContainerCpuLimit")]
61+
public string WaitContainerCpuLimit { get; set; } = "1";
62+
63+
[ConfigurationKeyName("waitContainerMemoryLimit")]
64+
public string WaitContainerMemoryLimit { get; set; } = "500Mi";
65+
66+
[ConfigurationKeyName("messageGeneratorContainerCpuLimit")]
67+
public string MessageGeneratorContainerCpuLimit { get; set; } = "1";
68+
69+
[ConfigurationKeyName("messageGeneratorContainerMemoryLimit")]
70+
public string MessageGeneratorContainerMemoryLimit { get; set; } = "500Mi";
71+
72+
[ConfigurationKeyName("messageSenderContainerCpuLimit")]
73+
public string MessageSenderContainerCpuLimit { get; set; } = "1";
74+
75+
[ConfigurationKeyName("messageSenderContainerMemoryLimit")]
76+
public string MessageSenderContainerMemoryLimit { get; set; } = "500Mi";
5377
}
5478
}

‎src/Shared/Configuration/WorkflowManagerOptions.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,11 @@ public class WorkflowManagerOptions
6060
/// </summary>
6161
public BackgroundServiceSettings BackgroundServiceSettings { get; set; }
6262

63-
[ConfigurationKeyName("argoTtlStatergySeconds")]
64-
public int ArgoTtlStatergySeconds { get; set; } = 60 * 60 * 24 * 2; // 2 days before the pods get automatically cleaned up from argo
63+
[ConfigurationKeyName("argoTtlStrategySeconds")]
64+
public int ArgoTtlStrategySeconds { get; set; } = 60 * 60 * 24 * 2; // 2 days before the pods get automatically cleaned up from argo
6565

66-
[ConfigurationKeyName("minArgoTtlStatergySeconds")]
67-
public int MinArgoTtlStatergySeconds { get; set; } = 30; // time to get logs before cleanup !
66+
[ConfigurationKeyName("minArgoTtlStrategySeconds")]
67+
public int MinArgoTtlStrategySeconds { get; set; } = 30; // time to get logs before cleanup !
6868

6969
public WorkflowManagerOptions()
7070
{

‎src/TaskManager/API/Migrations/DocumentVersionConvert.cs

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright 2022 MONAI Consortium
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
using Mongo.Migration.Documents;
18+
using Newtonsoft.Json;
19+
20+
namespace Monai.Deploy.WorkflowManager.TaskManager.Migrations
21+
{
22+
public class DocumentVersionConvert : JsonConverter
23+
{
24+
public override bool CanConvert(Type objectType) => GetType() == objectType;
25+
public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
26+
{
27+
var major = 1;
28+
var minor = 0;
29+
var revision = 0;
30+
31+
var res = (reader.Value as string)?.Split(new char[] { ',' });
32+
33+
if (res?.Length == 3)
34+
{
35+
major = Convert.ToInt32(res[0]);
36+
minor = Convert.ToInt32(res[1]);
37+
revision = Convert.ToInt32(res[2]);
38+
}
39+
return new DocumentVersion(major, minor, revision);
40+
41+
}
42+
public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer)
43+
{
44+
serializer.Serialize(writer, value?.ToString());
45+
}
46+
}
47+
}

0 commit comments

Comments
 (0)