Skip to content

Commit 845b579

Browse files
MiraGeowerkstattMiraGeowerkstatt
committed
Make ID Import dynamic
1 parent 13b7c8f commit 845b579

File tree

7 files changed

+43
-40
lines changed

7 files changed

+43
-40
lines changed

‎src/api/Controllers/ImportController.cs‎

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,14 @@ public async Task<ActionResult<int>> UploadFileAsync(int workgroupId, IFormFile
146146
// Checks if the provided boreholes file is a CSV file.
147147
if (!FileTypeChecker.IsCsv(boreholesFile)) return BadRequest("Invalid file type for borehole csv.");
148148

149-
var boreholeImports = ReadBoreholesFromCsv(boreholesFile);
149+
// The identifier codelists are used to dynamically map imported identifiers to codelists.
150+
var identifierCodelists = await context.Codelists
151+
.Where(c => c.Schema == "borehole_identifier")
152+
.AsNoTracking()
153+
.ToListAsync()
154+
.ConfigureAwait(false);
155+
156+
var boreholeImports = ReadBoreholesFromCsv(boreholesFile, identifierCodelists);
150157
ValidateBoreholeImports(workgroupId, boreholeImports, false);
151158

152159
// If any validation error occured, return a bad request.
@@ -307,12 +314,12 @@ internal static bool CompareValuesWithTolerance(double? firstValue, double? seco
307314
return Math.Abs(firstValue.Value - secondValue.Value) <= tolerance;
308315
}
309316

310-
private static List<BoreholeImport> ReadBoreholesFromCsv(IFormFile file)
317+
private static List<BoreholeImport> ReadBoreholesFromCsv(IFormFile file, List<Codelist> identifierCodelists)
311318
{
312319
using var reader = new StreamReader(file.OpenReadStream());
313320
using var csv = new CsvReader(reader, CsvConfigHelper.CsvReadConfig);
314321

315-
csv.Context.RegisterClassMap(new CsvImportBoreholeMap());
322+
csv.Context.RegisterClassMap(new CsvImportBoreholeMap(identifierCodelists));
316323

317324
return csv.GetRecords<BoreholeImport>().ToList();
318325
}
@@ -348,8 +355,11 @@ private void AddValidationErrorToModelState(int boreholeIndex, string errorMessa
348355

349356
private sealed class CsvImportBoreholeMap : ClassMap<BoreholeImport>
350357
{
351-
public CsvImportBoreholeMap()
358+
private readonly List<Codelist> codelists;
359+
360+
public CsvImportBoreholeMap(List<Codelist> codelists)
352361
{
362+
this.codelists = codelists;
353363
AutoMap(CsvConfigHelper.CsvReadConfig);
354364

355365
// Define all optional properties of Borehole (ef navigation properties do not need to be defined as optional).
@@ -401,37 +411,31 @@ public CsvImportBoreholeMap()
401411
Map(m => m.TopBedrockFreshTvd).Ignore();
402412
Map(m => m.TopBedrockWeatheredTvd).Ignore();
403413

404-
// Define additional mapping logic
405414
Map(m => m.BoreholeCodelists).Convert(args =>
406415
{
407416
var boreholeCodeLists = new List<BoreholeCodelist>();
408-
new List<(string Name, int CodeListId)>
409-
{
410-
("IDGeODin-Shortname", 100000000),
411-
("IDInfoGeol", 100000003),
412-
("IDOriginal", 100000004),
413-
("IDCanton", 100000005),
414-
("IDGeoQuat", 100000006),
415-
("IDGeoMol", 100000007),
416-
("IDGeoTherm", 100000008),
417-
("IDTopFels", 100000009),
418-
("IDGeODin", 100000010),
419-
("IDKernlager", 100000011),
420-
}.ForEach(id =>
417+
418+
foreach (var header in args.Row.HeaderRecord ?? Array.Empty<string>())
421419
{
422-
if (args.Row.HeaderRecord != null && args.Row.HeaderRecord.Any(h => h == id.Name))
420+
// Find the corresponding codelist by comparing the header with Codelist.En, ignoring whitespace
421+
var codelist = codelists.FirstOrDefault(cl => string.Equals(
422+
cl.En.Replace(" ", string.Empty, StringComparison.OrdinalIgnoreCase),
423+
header.Replace(" ", string.Empty, StringComparison.OrdinalIgnoreCase),
424+
StringComparison.OrdinalIgnoreCase));
425+
426+
if (codelist != null)
423427
{
424-
var value = args.Row.GetField<string?>(id.Name);
428+
var value = args.Row.GetField<string?>(header);
425429
if (!string.IsNullOrEmpty(value))
426430
{
427431
boreholeCodeLists.Add(new BoreholeCodelist
428432
{
429-
CodelistId = id.CodeListId,
433+
CodelistId = codelist.Id,
430434
Value = value,
431435
});
432436
}
433437
}
434-
});
438+
}
435439

436440
return boreholeCodeLists;
437441
});

‎src/api/Models/Codelist.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.ComponentModel.DataAnnotations;
33
using System.ComponentModel.DataAnnotations.Schema;
44
using System.Text.Json.Serialization;
5+
using System.Text.Json;
56

67
namespace BDMS.Models;
78

@@ -74,7 +75,7 @@ public class Codelist : IIdentifyable
7475
/// Gets or sets the <see cref="Codelist"/>'s configuration.
7576
/// </summary>
7677
[Column("conf_cli")]
77-
public string? Conf { get; set; }
78+
public JsonDocument? Conf { get; set; }
7879

7980
/// <summary>
8081
/// Gets or sets whether the <see cref="Codelist"/> is default.

‎src/client/docs/import.md‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,6 @@ Die zu importierenden Daten müssen gemäss obigen Anforderungen im CSV-Format v
4040

4141
| Feldname | Datentyp | Pflichtfeld | Beschreibung |
4242
| --------------------------- | -------------- | ----------- | ------------------------------------------------------------------------------------- |
43-
| IDGeODin-Shortname | Zahl | Nein | ID GeODin-Shortname |
44-
| IDInfoGeol | Zahl | Nein | ID InfoGeol |
45-
| IDOriginal | Zahl | Nein | ID Original |
46-
| IDCanton | Zahl | Nein | ID Kanton |
47-
| IDGeoQuat | Zahl | Nein | ID GeoQuat |
48-
| IDGeoMol | Zahl | Nein | ID GeoMol |
49-
| IDGeoTherm | Zahl | Nein | ID GeoTherm |
50-
| IDTopFels | Zahl | Nein | ID TopFels |
51-
| IDGeODin | Zahl | Nein | ID GeODin |
52-
| IDKernlager | Zahl | Nein | ID Kernlager |
5343
| OriginalName | Text | Ja | Originalname |
5444
| ProjectName | Text | Nein | Projektname |
5545
| Name | Text | Nein | Name |
@@ -78,6 +68,10 @@ Die zu importierenden Daten müssen gemäss obigen Anforderungen im CSV-Format v
7868
| ChronostratigraphyTopBedrockId| ID (Codeliste) | Nein | Chronostratigraphie Top Fels |
7969
| LithostratigraphyTopBedrockId | ID (Codeliste) | Nein | Lithostratigraphie Top Fels |
8070

71+
### Ids
72+
Es können zusätzliche IDs importiert werden. Die dafür zu verwendenden Spaltenüberschriften sind dynamisch und können von Umgebung zu Umgebung variieren.
73+
Um die korrekten Spaltenüberschriften zu erhalten, kann eine Bohrung mit einer entsprechenden ID als CSV-Datei exportiert werden.
74+
8175
### Koordinaten
8276

8377
Koordinaten können in LV95 oder LV03 importiert werden, das räumliche Bezugssystem wird aus den Koordinaten erkannt und abgespeichert.

‎src/client/src/pages/overview/sidePanelContent/importer/importModalContent.tsx‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,7 @@ const ImportModalContent = ({ setSelectedFile, setFileType, fileType }: ImportMo
6464
<StackHalfWidth direction="column">
6565
{t("csvFormatExplanation")}
6666
{ExampleHeadings(
67-
"IdOriginal;" +
68-
"IdCanton;IdGeoQuat;IdGeoMol;IdGeoTherm;IdTopFels;" +
69-
"IdGeodin;IdKernlager;OriginalName;ProjectName;Name;" +
67+
"OriginalName;ProjectName;Name;" +
7068
"RestrictionId;RestrictionUntil;NationalInterest;LocationX;LocationY;" +
7169
"LocationPrecision;ElevationZ;ElevationPrecisionId;" +
7270
"ReferenceElevation;ReferenceElevationTypeId;" +

‎tests/api/Controllers/CodeListControllerTest.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public async Task GetEntriesBySchema()
5151
var codeListToTest = codeLists.Single(c => c.Id == 15001070);
5252
Assert.AreEqual(15001070, codeListToTest.Id);
5353
Assert.AreEqual(15001070, codeListToTest.Geolcode);
54-
Assert.AreEqual("{\"color\":[128,207,216]}", codeListToTest.Conf);
54+
Assert.AreEqual("{\"color\":[128,207,216]}", codeListToTest.Conf.RootElement.ToString());
5555
Assert.AreEqual(false, codeListToTest.IsDefault);
5656
Assert.AreEqual("custom.chronostratigraphy_top_bedrock", codeListToTest.Schema);
5757
Assert.AreEqual("Mittlerer Jura", codeListToTest.De);

‎tests/api/Controllers/ImportControllerTest.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public class ImportControllerTest
2121
private const int MaxBoreholeSeedId = 1002999;
2222
private const int MaxStratigraphySeedId = 6002999;
2323
private const int MaxLayerSeedId = 7029999;
24+
private const int TestCodelistId = 955253;
2425

2526
private BdmsContext context;
2627
private ImportController controller;
@@ -71,6 +72,7 @@ public async Task TestCleanup()
7172
context.Workflows.RemoveRange(addedWorkflows);
7273
context.Stratigraphies.RemoveRange(addedStratigraphies);
7374
context.Layers.RemoveRange(addedLayers);
75+
context.Codelists.RemoveRange(context.Codelists.Where(c => c.Id == TestCodelistId));
7476
context.SaveChanges();
7577

7678
await context.DisposeAsync();
@@ -511,6 +513,9 @@ public async Task UploadJsonWithDuplicatesExistingBoreholeShouldReturnError()
511513
[TestMethod]
512514
public async Task UploadShouldSaveDataToDatabaseAsync()
513515
{
516+
context.Codelists.Add(new Codelist { Id = TestCodelistId, Schema = "borehole_identifier", Code = "new code", En = "Random New Id", Conf = null });
517+
await context.SaveChangesAsync();
518+
514519
httpClientFactoryMock
515520
.Setup(cf => cf.CreateClient(It.IsAny<string>()))
516521
.Returns(() => new HttpClient())
@@ -532,9 +537,10 @@ public async Task UploadShouldSaveDataToDatabaseAsync()
532537
Assert.AreEqual(new DateTime(2024, 06, 15), borehole.RestrictionUntil);
533538
Assert.AreEqual(2474.472693, borehole.TotalDepth);
534539
Assert.AreEqual("Projekt 6", borehole.ProjectName);
535-
Assert.AreEqual(4, borehole.BoreholeCodelists.Count);
540+
Assert.AreEqual(5, borehole.BoreholeCodelists.Count);
536541
Assert.AreEqual("Id_16", borehole.BoreholeCodelists.Single(x => x.CodelistId == 100000003).Value);
537542
Assert.AreEqual("AUTOSTEED", borehole.BoreholeCodelists.Single(x => x.CodelistId == 100000011).Value);
543+
Assert.AreEqual("121314", borehole.BoreholeCodelists.Single(x => x.CodelistId == TestCodelistId).Value);
538544
Assert.AreEqual("Bern", borehole.Canton);
539545
Assert.AreEqual("Schweiz", borehole.Country);
540546
Assert.AreEqual("Thun", borehole.Municipality);

‎tests/api/TestData/testdata.csv‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
IDGeODin-Shortname;IDInfoGeol;IDOriginal;IDCanton;IDGeoQuat;IDGeoMol;IDGeoTherm;IDTopFels;IDGeODin;IDKernlager;original_name;project_name;name;date;restriction_id;restriction_until;original_reference_system;location_x;location_y;location_x_lv_03;location_y_lv_03;location_precision_id;elevation_z;elevation_precision_id;reference_elevation;reference_elevation_type_id;reference_elevation_precision_id;hrs_id;type_id;purpose_id;status_id;remarks;total_depth;depth_precision_id;top_bedrock_fresh_md;top_bedrock_weathered_md;has_groundwater;lithology_top_bedrock_id;chronostratigraphy_top_bedrock_id;lithostratigraphy_top_bedrock_id
1+
IDGeODin-Shortname;IDInfoGeol;IDOriginal;IDCanton;IDGeoQuat;IDGeoMol;IDGeoTherm;IDTopFels;IDGeODin;IDKernlager;original_name;project_name;name;date;restriction_id;restriction_until;original_reference_system;location_x;location_y;location_x_lv_03;location_y_lv_03;location_precision_id;elevation_z;elevation_precision_id;reference_elevation;reference_elevation_type_id;reference_elevation_precision_id;hrs_id;type_id;purpose_id;status_id;remarks;total_depth;depth_precision_id;top_bedrock_fresh_md;top_bedrock_weathered_md;has_groundwater;lithology_top_bedrock_id;chronostratigraphy_top_bedrock_id;lithostratigraphy_top_bedrock_id;RandomNewId
22
Id_1;Id_2;;;;;Id_3;;;kernlager AETHERMAGIC;Unit_Test_1;Projekt 1 ;Unit_Test_1_a;2021-08-06 00:36:21.991827+00;20111002;;20104001;2618962;1144995;;;20113005;640.7726659;20114001;317.9010264;20117002;20114004;20106001;20101001;22103001;22104003;this product is top-notch.;4232.711946;22108003;398.8529283;656.2476436;TRUE;15104669;15001073;15300261
33
Id_4;;Id_5;Id_6;;;;;;;Unit_Test_2;Projekt 2;Unit_Test_2_a;2021-03-31 12:20:10.341393+00;;;20104001;2631690;1170516;;;20113002;3430.769638;20114005;2016.314814;20117005;20114004;20106001;20101001;22103001;22104008;This product works certainly well. It perfectly improves my tennis by a lot.;794.1547194;22108005;958.2378855;549.9801019;;15104670;15001009;15302009
44
;;Id_7;Id_8;;;;Id_9;;;Unit_Test_3;Projekt 3;Unit_Test_3_a;;20111002;01.12.2023;20104001;2614834;1178661;;;20113005;1720.766609;20114003;1829.812475;20117005;20114002;20106001;20101001;;22104002;This is a really good product.;2429.747725;22108002;759.7574008;827.8441205;TRUE;15104671;15001007;15302339
55
Id_10;;;Id_11;Id_12;;;;;;Unit_Test_4;Projekt 4;Unit_Test_4_a;01.12.2023;;;20104002;2599840;1200560;;;20113004;10.76358115;20114004;1260.544983;20117004;20114001;20106001;20101001;22103001;22104001;;4077.768394;22108004;656.2476436;398.8529283;FALSE;15104672;15001064;15302017
66
Id_13;;;;Id_14;;;;Id_15;;Unit_Test_5;Projekt 5;Unit_Test_5_a;2021-03-13 23:31:35.390094+00;;;20104001;2631718;1170532;;;20113002;2800.760553;20114004;1928.223082;20117006;20114001;20106001;20101001;22103001;22104008;I tried to maim it but got nectarine all over it.;2971.608569;22108005;549.9801019;958.2378855;FALSE;15104673;15001139;
7-
;Id_16;;;;Id_17;Id_18;;;AUTOSTEED;Unit_Test_6;Projekt 6;Unit_Test_6_a;2021-11-06 11:15:23.73966+00;20111003;15.06.2024;20104001;2613116;1179127;;;20113004;1090.757525;20114005;1829.812475;20117001;20114001;20106001;20101001;22103004;22104003;talk about fury.;2474.472693;22108005;827.8441205;759.7574008;TRUE;15104674;15001006;15302267
7+
;Id_16;;;;Id_17;Id_18;;;AUTOSTEED;Unit_Test_6;Projekt 6;Unit_Test_6_a;2021-11-06 11:15:23.73966+00;20111003;15.06.2024;20104001;2613116;1179127;;;20113004;1090.757525;20114005;1829.812475;20117001;20114001;20106001;20101001;22103004;22104003;talk about fury.;2474.472693;22108005;827.8441205;759.7574008;TRUE;15104674;15001006;15302267;121314

0 commit comments

Comments
 (0)