Json export endpoint (#1774)

CHANGELOG.md

@@ -7,6 +7,10 @@
 - Updated recommended csv headers for borehole import to camel case e.g. `OriginalName` (snake case e.g. `original_name` is still supported for all properties except for custom identifiers).
 - Changed order of `Top Bedrock (fresh)` and `Top Bedrock (weathered)` fields in borehole form.
 
+### Fixed
+
+- Observations were not included in exported borehole JSON file.
+
 ## v2.1.993 - 2024-12-13
 
 ### Added

src/api/BoreholeExtensions.cs

@@ -0,0 +1,36 @@
+using BDMS.Models;
+using Microsoft.EntityFrameworkCore;
+
+namespace BDMS;
+
+public static class BoreholeExtensions
+{
+    public static IQueryable<Borehole> GetAllWithIncludes(this DbSet<Borehole> boreholes)
+    {
+        return boreholes.Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerColorCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerDebrisCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerGrainAngularityCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerGrainShapeCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerOrganicComponentCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerUscs3Codes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.LithologicalDescriptions)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.FaciesDescriptions)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.ChronostratigraphyLayers)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.LithostratigraphyLayers)
+            .Include(b => b.Completions).ThenInclude(c => c.Casings).ThenInclude(c => c.CasingElements)
+            .Include(b => b.Completions).ThenInclude(c => c.Instrumentations)
+            .Include(b => b.Completions).ThenInclude(c => c.Backfills)
+            .Include(b => b.Sections).ThenInclude(s => s.SectionElements)
+            .Include(b => b.Observations).ThenInclude(o => (o as FieldMeasurement)!.FieldMeasurementResults)
+            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestResults)
+            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestEvaluationMethodCodes)
+            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestFlowDirectionCodes)
+            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestKindCodes)
+            .Include(b => b.BoreholeCodelists)
+            .Include(b => b.Workflows)
+            .Include(b => b.BoreholeFiles)
+            .Include(b => b.BoreholeGeometry)
+            .Include(b => b.Workgroup)
+            .Include(b => b.UpdatedBy);
+    }
+}

src/api/Controllers/BoreholeController.cs

@@ -1,13 +1,10 @@
 using BDMS.Authentication;
 using BDMS.Models;
-using CsvHelper;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using NetTopologySuite.Geometries;
 using System.ComponentModel.DataAnnotations;
-using System.Globalization;
-using System.Text;
 
 namespace BDMS.Controllers;
 
@@ -85,7 +82,7 @@ public async Task<ActionResult<PaginatedBoreholeResponse>> GetAllAsync([FromQuer
         pageSize = Math.Min(MaxPageSize, Math.Max(1, pageSize));
 
         var skip = (pageNumber - 1) * pageSize;
-        var query = GetBoreholesWithIncludes().AsNoTracking();
+        var query = Context.Boreholes.GetAllWithIncludes().AsNoTracking();
 
         if (ids != null && ids.Any())
         {
@@ -107,7 +104,7 @@ public async Task<ActionResult<PaginatedBoreholeResponse>> GetAllAsync([FromQuer
     [Authorize(Policy = PolicyNames.Viewer)]
     public async Task<ActionResult<Borehole>> GetByIdAsync(int id)
     {
-        var borehole = await GetBoreholesWithIncludes()
+        var borehole = await Context.Boreholes.GetAllWithIncludes()
             .AsNoTracking()
             .SingleOrDefaultAsync(l => l.Id == id)
             .ConfigureAwait(false);
@@ -143,7 +140,7 @@ public async Task<ActionResult<int>> CopyAsync([Required] int id, [Required] int
             return Unauthorized();
         }
 
-        var borehole = await GetBoreholesWithIncludes()
+        var borehole = await Context.Boreholes.GetAllWithIncludes()
             .AsNoTracking()
             .SingleOrDefaultAsync(b => b.Id == id)
             .ConfigureAwait(false);
@@ -255,33 +252,4 @@ public async Task<ActionResult<int>> CopyAsync([Required] int id, [Required] int
         if (entity == null) return default;
         return await Task.FromResult<int?>(entity.Id).ConfigureAwait(false);
     }
-
-    private IQueryable<Borehole> GetBoreholesWithIncludes()
-    {
-        return Context.Boreholes.Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerColorCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerDebrisCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerGrainAngularityCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerGrainShapeCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerOrganicComponentCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerUscs3Codes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.LithologicalDescriptions)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.FaciesDescriptions)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.ChronostratigraphyLayers)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.LithostratigraphyLayers)
-            .Include(b => b.Completions).ThenInclude(c => c.Casings).ThenInclude(c => c.CasingElements)
-            .Include(b => b.Completions).ThenInclude(c => c.Instrumentations)
-            .Include(b => b.Completions).ThenInclude(c => c.Backfills)
-            .Include(b => b.Sections).ThenInclude(s => s.SectionElements)
-            .Include(b => b.Observations).ThenInclude(o => (o as FieldMeasurement)!.FieldMeasurementResults)
-            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestResults)
-            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestEvaluationMethodCodes)
-            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestFlowDirectionCodes)
-            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestKindCodes)
-            .Include(b => b.BoreholeCodelists)
-            .Include(b => b.Workflows)
-            .Include(b => b.BoreholeFiles)
-            .Include(b => b.BoreholeGeometry)
-            .Include(b => b.Workgroup)
-            .Include(b => b.UpdatedBy);
-    }
 }

src/api/Controllers/ExportController.cs

@@ -6,6 +6,8 @@
 using Microsoft.EntityFrameworkCore;
 using System.ComponentModel.DataAnnotations;
 using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace BDMS.Controllers;
 
@@ -18,11 +20,33 @@ public class ExportController : ControllerBase
     private const int MaxPageSize = 100;
     private readonly BdmsContext context;
 
+    private static readonly JsonSerializerOptions jsonExportOptions = new()
+    {
+        WriteIndented = true,
+        ReferenceHandler = ReferenceHandler.IgnoreCycles,
+        Converters = { new DateOnlyJsonConverter(), new LTreeJsonConverter(), new ObservationConverter() },
+    };
+
     public ExportController(BdmsContext context)
     {
         this.context = context;
     }
 
+    /// <summary>
+    /// Asynchronously gets all <see cref="Borehole"/> records filtered by ids. Additional data is included in the response.
+    /// </summary>
+    /// <param name="ids">The required list of borehole ids to filter by.</param>
+    [HttpGet("json")]
+    [Authorize(Policy = PolicyNames.Viewer)]
+    public async Task<ActionResult> ExportJsonAsync([FromQuery][MaxLength(MaxPageSize)] IEnumerable<int> ids)
+    {
+        if (ids == null || !ids.Any()) return BadRequest("The list of IDs must not be empty.");
+
+        var boreholes = await context.Boreholes.GetAllWithIncludes().AsNoTracking().Where(borehole => ids.Contains(borehole.Id)).ToListAsync().ConfigureAwait(false);
+
+        return new JsonResult(boreholes, jsonExportOptions);
+    }
+
     /// <summary>
     /// Exports the details of up to <see cref="MaxPageSize"></see> boreholes as a CSV file. Filters the boreholes based on the provided list of IDs.
     /// </summary>

src/api/Controllers/ImportController.cs

@@ -9,6 +9,7 @@
 using System.Globalization;
 using System.Net;
 using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace BDMS.Controllers;
 
@@ -26,7 +27,12 @@ public class ImportController : ControllerBase
     private readonly int sridLv03 = 21781;
     private readonly string nullOrEmptyMsg = "Field '{0}' is required.";
 
-    private static readonly JsonSerializerOptions jsonImportOptions = new() { PropertyNameCaseInsensitive = true };
+    private static readonly JsonSerializerOptions jsonImportOptions = new()
+    {
+        PropertyNameCaseInsensitive = true,
+        ReferenceHandler = ReferenceHandler.IgnoreCycles,
+        Converters = { new DateOnlyJsonConverter(), new LTreeJsonConverter(), new ObservationConverter() },
+    };
 
     public ImportController(BdmsContext context, ILogger<ImportController> logger, LocationService locationService, CoordinateService coordinateService, BoreholeFileCloudService boreholeFileCloudService)
     {
@@ -89,8 +95,12 @@ public async Task<ActionResult<int>> UploadJsonFileAsync(int workgroupId, IFormF
             foreach (var borehole in boreholes)
             {
                 borehole.MarkAsNew();
+                borehole.Workgroup = null;
                 borehole.WorkgroupId = workgroupId;
+                borehole.LockedBy = null;
                 borehole.LockedById = null;
+                borehole.UpdatedBy = null;
+                borehole.CreatedBy = null;
 
                 borehole.Stratigraphies?.MarkAsNew();
                 borehole.Completions?.MarkAsNew();
@@ -99,7 +109,7 @@ public async Task<ActionResult<int>> UploadJsonFileAsync(int workgroupId, IFormF
 
                 // Do not import any workflows from the json file but add a new unfinished workflow for the current user.
                 borehole.Workflows.Clear();
-                borehole.Workflows.Add(new Workflow { Borehole = borehole, Role = Role.Editor, UserId = user.Id, Started = DateTime.Now.ToUniversalTime() });
+                borehole.Workflows.Add(new Workflow { Role = Role.Editor, UserId = user.Id, Started = DateTime.Now.ToUniversalTime() });
             }
 
             await context.Boreholes.AddRangeAsync(boreholes).ConfigureAwait(false);

src/api/Controllers/ObservationConverter.cs

@@ -0,0 +1,64 @@
+using BDMS.Models;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace BDMS.Controllers;
+
+/// <summary>
+/// Serializes and deserializes <see cref="Observation"/> objects based on their ObservationType.
+/// </summary>
+public class ObservationConverter : JsonConverter<Observation>
+{
+    private static readonly JsonSerializerOptions observationDefaultOptions = new JsonSerializerOptions
+    {
+        PropertyNameCaseInsensitive = true,
+        ReferenceHandler = ReferenceHandler.IgnoreCycles,
+    };
+
+    /// <inheritdoc/>
+    public override Observation? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        using JsonDocument doc = JsonDocument.ParseValue(ref reader);
+        var jsonObject = doc.RootElement;
+
+        // Deserialize the observation type to determine the observation type
+        var observation = JsonSerializer.Deserialize<Observation>(jsonObject.GetRawText(), observationDefaultOptions);
+
+        return observation.Type switch
+        {
+            ObservationType.Hydrotest => JsonSerializer.Deserialize<Hydrotest>(jsonObject.GetRawText(), options),
+            ObservationType.FieldMeasurement => JsonSerializer.Deserialize<FieldMeasurement>(jsonObject.GetRawText(), options),
+            ObservationType.WaterIngress => JsonSerializer.Deserialize<WaterIngress>(jsonObject.GetRawText(), options),
+            ObservationType.GroundwaterLevelMeasurement => JsonSerializer.Deserialize<GroundwaterLevelMeasurement>(jsonObject.GetRawText(), options),
+            _ => observation,
+        };
+    }
+
+    /// <inheritdoc/>
+    public override void Write(Utf8JsonWriter writer, Observation value, JsonSerializerOptions options)
+    {
+        switch (value)
+        {
+            case Hydrotest hydrotest:
+                hydrotest.EvaluationMethodCodelistIds = hydrotest.HydrotestEvaluationMethodCodes?.Select(x => x.CodelistId).ToList();
+                hydrotest.FlowDirectionCodelistIds = hydrotest.HydrotestFlowDirectionCodes?.Select(x => x.CodelistId).ToList();
+                hydrotest.KindCodelistIds = hydrotest.HydrotestKindCodes?.Select(x => x.CodelistId).ToList();
+                JsonSerializer.Serialize(writer, hydrotest, options);
+                break;
+            case FieldMeasurement fieldMeasurement:
+                JsonSerializer.Serialize(writer, fieldMeasurement, options);
+                break;
+            case WaterIngress waterIngress:
+                JsonSerializer.Serialize(writer, waterIngress, options);
+                break;
+            case GroundwaterLevelMeasurement groundwaterLevelMeasurement:
+                JsonSerializer.Serialize(writer, groundwaterLevelMeasurement, options);
+                break;
+            case Observation observation:
+                JsonSerializer.Serialize(writer, observation, options);
+                break;
+            default:
+                throw new NotSupportedException("Observation type is not supported");
+        }
+    }
+}

src/client/src/api/borehole.ts

@@ -67,6 +67,11 @@ export interface BoreholeV2 {
 
 export const getBoreholeById = async (id: number) => await fetchApiV2(`borehole/${id}`, "GET");
 
+export const exportJsonBoreholes = async (ids: number[] | GridRowSelectionModel) => {
+  const idsQuery = ids.map(id => `ids=${id}`).join("&");
+  return await fetchApiV2(`borehole/json?${idsQuery}`, "GET");
+};
+
 export const updateBorehole = async (borehole: BoreholeV2) => {
   return await fetchApiV2("borehole", "PUT", borehole);
 };

src/client/src/pages/detail/form/hydrogeology/hydrotestInput.jsx

@@ -157,7 +157,7 @@ const HydrotestInput = props => {
     data = prepareCasingDataForSubmit(data);
     data?.startTime ? (data.startTime += ":00.000Z") : (data.startTime = null);
     data?.endTime ? (data.endTime += ":00.000Z") : (data.endTime = null);
-    data.type = ObservationType.fieldMeasurement;
+    data.type = ObservationType.hydrotest;
     data.boreholeId = parentId;
 
     if (Array.isArray(data.testKindId)) {