Merge main

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## [Unreleased]
 
+### Changed 
+- Removed attachments from csv import.
+- Updated recommended csv headers for borehole import to camel case e.g. `OriginalName` (snake case e.g. `original_name` is still supported for all properties except for custom identifiers).
+- Changed order of `Top Bedrock (fresh)` and `Top Bedrock (weathered)` fields in borehole form.
+
+### Fixed
+
+- Observations were not included in exported borehole JSON file.
+
 ## v2.1.993 - 2024-12-13
 
 ### Added
@@ -47,7 +56,6 @@
 - When copying a borehole, the nested collections of observations were not copied.
 - There was a bug when changing the order, transparency or visibility of custom WMS user layers.
 - The borehole status was not translated everywhere in the workflow panel.
-- Observations were not included in exported borehole JSON files.
 
 ## v2.1.870 - 2024-09-27
 

src/api/BdmsContextExtensions.cs

@@ -202,6 +202,9 @@ public static void SeedData(this BdmsContext context)
            .RuleFor(o => o.PrecisionLocationY, f => f.PickRandom(Enumerable.Range(0, 10)))
            .RuleFor(o => o.PrecisionLocationXLV03, f => f.PickRandom(Enumerable.Range(0, 10)))
            .RuleFor(o => o.PrecisionLocationYLV03, f => f.PickRandom(Enumerable.Range(0, 10)))
+           .RuleFor(o => o.TotalDepthTvd, _ => null)
+           .RuleFor(o => o.TopBedrockFreshTvd, _ => null)
+           .RuleFor(o => o.TopBedrockWeatheredTvd, _ => null)
            .RuleFor(o => o.Observations, _ => new Collection<Observation>())
            .FinishWith((f, o) => { o.Name = o.OriginalName; });
 

src/api/BoreholeExtensions.cs

@@ -0,0 +1,36 @@
+using BDMS.Models;
+using Microsoft.EntityFrameworkCore;
+
+namespace BDMS;
+
+public static class BoreholeExtensions
+{
+    public static IQueryable<Borehole> GetAllWithIncludes(this DbSet<Borehole> boreholes)
+    {
+        return boreholes.Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerColorCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerDebrisCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerGrainAngularityCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerGrainShapeCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerOrganicComponentCodes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerUscs3Codes)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.LithologicalDescriptions)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.FaciesDescriptions)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.ChronostratigraphyLayers)
+            .Include(b => b.Stratigraphies).ThenInclude(s => s.LithostratigraphyLayers)
+            .Include(b => b.Completions).ThenInclude(c => c.Casings).ThenInclude(c => c.CasingElements)
+            .Include(b => b.Completions).ThenInclude(c => c.Instrumentations)
+            .Include(b => b.Completions).ThenInclude(c => c.Backfills)
+            .Include(b => b.Sections).ThenInclude(s => s.SectionElements)
+            .Include(b => b.Observations).ThenInclude(o => (o as FieldMeasurement)!.FieldMeasurementResults)
+            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestResults)
+            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestEvaluationMethodCodes)
+            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestFlowDirectionCodes)
+            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestKindCodes)
+            .Include(b => b.BoreholeCodelists)
+            .Include(b => b.Workflows)
+            .Include(b => b.BoreholeFiles)
+            .Include(b => b.BoreholeGeometry)
+            .Include(b => b.Workgroup)
+            .Include(b => b.UpdatedBy);
+    }
+}

src/api/BoreholeGeometry/AzIncFormat.cs

@@ -21,7 +21,7 @@ internal sealed class AzIncFormat : IBoreholeGeometryFormat
     public IList<BoreholeGeometryElement> ReadCsv(IFormFile file, int boreholeId)
     {
         using var reader = new StreamReader(file.OpenReadStream());
-        using var csv = new CsvReader(reader, CsvConfigHelper.CsvConfig);
+        using var csv = new CsvReader(reader, CsvConfigHelper.CsvReadConfig);
 
         var data = csv.GetRecords<Geometry>().ToList();
 

src/api/BoreholeGeometry/PitchRollFormat.cs

@@ -21,7 +21,7 @@ internal sealed class PitchRollFormat : IBoreholeGeometryFormat
     public IList<BoreholeGeometryElement> ReadCsv(IFormFile file, int boreholeId)
     {
         using var reader = new StreamReader(file.OpenReadStream());
-        using var csv = new CsvReader(reader, CsvConfigHelper.CsvConfig);
+        using var csv = new CsvReader(reader, CsvConfigHelper.CsvReadConfig);
 
         var data = csv.GetRecords<Geometry>().ToList();
 

src/api/BoreholeGeometry/XYZFormat.cs

@@ -20,7 +20,7 @@ internal sealed class XYZFormat : IBoreholeGeometryFormat
     public IList<BoreholeGeometryElement> ReadCsv(IFormFile file, int boreholeId)
     {
         using var reader = new StreamReader(file.OpenReadStream());
-        using var csv = new CsvReader(reader, CsvConfigHelper.CsvConfig);
+        using var csv = new CsvReader(reader, CsvConfigHelper.CsvReadConfig);
 
         var data = csv.GetRecords<Geometry>();
         return ToBoreholeGeometry(data, boreholeId);

src/api/Controllers/BoreholeController.cs

@@ -1,17 +1,10 @@
 using BDMS.Authentication;
 using BDMS.Models;
-using CsvHelper;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using NetTopologySuite.Geometries;
 using System.ComponentModel.DataAnnotations;
-using System.Globalization;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using OSGeo.OGR;
-using MaxRev.Gdal.Core;
 
 namespace BDMS.Controllers;
 
@@ -89,7 +82,7 @@ public async Task<ActionResult<PaginatedBoreholeResponse>> GetAllAsync([FromQuer
         pageSize = Math.Min(MaxPageSize, Math.Max(1, pageSize));
 
         var skip = (pageNumber - 1) * pageSize;
-        var query = GetBoreholesWithIncludes().AsNoTracking();
+        var query = Context.Boreholes.GetAllWithIncludes().AsNoTracking();
 
         if (ids != null && ids.Any())
         {
@@ -111,7 +104,7 @@ public async Task<ActionResult<PaginatedBoreholeResponse>> GetAllAsync([FromQuer
     [Authorize(Policy = PolicyNames.Viewer)]
     public async Task<ActionResult<Borehole>> GetByIdAsync(int id)
     {
-        var borehole = await GetBoreholesWithIncludes()
+        var borehole = await Context.Boreholes.GetAllWithIncludes()
             .AsNoTracking()
             .SingleOrDefaultAsync(l => l.Id == id)
             .ConfigureAwait(false);
@@ -124,171 +117,6 @@ public async Task<ActionResult<Borehole>> GetByIdAsync(int id)
         return Ok(borehole);
     }
 
-    /// <summary>
-    /// Asynchronously gets all <see cref="Borehole"/> records filtered by ids. Additional data is included in the response.
-    /// </summary>
-    /// <param name="ids">The required list of borehole ids to filter by.</param>
-    [HttpGet("json")]
-    [Authorize(Policy = PolicyNames.Viewer)]
-    public async Task<ActionResult> ExportJsonAsync([FromQuery][MaxLength(MaxPageSize)] IEnumerable<int> ids)
-    {
-        if (ids == null || !ids.Any())
-        {
-            return BadRequest("The list of IDs must not be empty.");
-        }
-
-        var boreholes = await GetBoreholesWithIncludes().AsNoTracking().Where(borehole => ids.Contains(borehole.Id)).ToListAsync().ConfigureAwait(false);
-
-        // Create a new JsonSerializerOptions for this specific endpoint
-        var options = new JsonSerializerOptions()
-        {
-            ReferenceHandler = ReferenceHandler.IgnoreCycles,
-            WriteIndented = true,
-        };
-
-        // Add the default converters from the global configuration
-        options.Converters.Add(new DateOnlyJsonConverter());
-        options.Converters.Add(new LTreeJsonConverter());
-
-        // Add special converter for the 'Observations' collection
-        options.Converters.Add(new ObservationConverter());
-
-        try
-        {
-            var features = boreholes.Select(borehole =>
-            {
-
-                var feature = new
-                {
-                    type = "Feature",
-                    geometry = new
-                    {
-                        type = "Point",
-                        crs = new
-                        {
-                            type = "name",
-                            properties = new
-                            {
-                                name = "EPSG:2056",
-                            },
-                        },
-                        coordinates = new[]
-                        {
-                            borehole.LocationX,
-                            borehole.LocationY,
-                        },
-                    },
-                    properties = borehole,
-                };
-                return feature;
-            }).ToList();
-
-            var geojson = new
-            {
-                type = "FeatureCollection",
-                crs = new
-                {
-                    type = "name",
-                    properties = new
-                    {
-                        name = "EPSG:2056",
-                    },
-                },
-                features,
-            };
-
-            var geojsonFile = "bulkexport_test.geojson";
-            await System.IO.File.WriteAllTextAsync(geojsonFile, JsonSerializer.Serialize(geojson, options)).ConfigureAwait(false);
-
-            GdalBase.ConfigureAll();
-            Ogr.RegisterAll();
-            Ogr.UseExceptions();
-
-            var geojsonDataSource = Ogr.Open(geojsonFile, 1);
-            if (geojsonDataSource == null)
-            {
-                throw new InvalidOperationException("Could not open input datasource.");
-            }
-
-            var gpkgFilePath = geojsonFile.Replace(".geojson", ".gpkg", StringComparison.InvariantCulture);
-            if (Directory.Exists(gpkgFilePath))
-            {
-                Directory.Delete(gpkgFilePath, true);
-            }
-
-            var openFileGdbDriver = Ogr.GetDriverByName("GPKG");
-            var gpkgDataSource = openFileGdbDriver.CreateDataSource(gpkgFilePath, null);
-
-            gpkgDataSource.CopyLayer(geojsonDataSource.GetLayerByIndex(0), "boreholes", null);
-
-            geojsonDataSource.Dispose();
-            gpkgDataSource.Dispose();
-        }
-        catch (Exception e)
-        {
-            Logger.LogError(e.Message);
-        }
-
-        return new JsonResult(boreholes, options);
-    }
-
-    /// <summary>
-    /// Exports the details of up to <see cref="MaxPageSize"></see> boreholes as a CSV file. Filters the boreholes based on the provided list of IDs.
-    /// </summary>
-    /// <param name="ids">The list of IDs for the boreholes to be exported.</param>
-    /// <returns>A CSV file containing the details specified boreholes.</returns>
-    [HttpGet("export-csv")]
-    [Authorize(Policy = PolicyNames.Viewer)]
-    public async Task<IActionResult> DownloadCsvAsync([FromQuery][MaxLength(MaxPageSize)] IEnumerable<int> ids)
-    {
-        ids = ids.Take(MaxPageSize).ToList();
-        if (!ids.Any()) return BadRequest("The list of IDs must not be empty.");
-
-        var boreholes = await Context.Boreholes
-            .Where(borehole => ids.Contains(borehole.Id))
-            .Select(b => new
-            {
-                b.Id,
-                b.OriginalName,
-                b.ProjectName,
-                b.Name,
-                b.RestrictionId,
-                b.RestrictionUntil,
-                b.NationalInterest,
-                b.LocationX,
-                b.LocationY,
-                b.LocationPrecisionId,
-                b.ElevationZ,
-                b.ElevationPrecisionId,
-                b.ReferenceElevation,
-                b.ReferenceElevationTypeId,
-                b.ReferenceElevationPrecisionId,
-                b.HrsId,
-                b.TypeId,
-                b.PurposeId,
-                b.StatusId,
-                b.Remarks,
-                b.TotalDepth,
-                b.DepthPrecisionId,
-                b.TopBedrockFreshMd,
-                b.TopBedrockWeatheredMd,
-                b.HasGroundwater,
-                b.LithologyTopBedrockId,
-                b.ChronostratigraphyTopBedrockId,
-                b.LithostratigraphyTopBedrockId,
-            })
-            .ToListAsync()
-            .ConfigureAwait(false);
-
-        if (boreholes.Count == 0) return NotFound("No borehole(s) found for the provided id(s).");
-
-        using var stringWriter = new StringWriter();
-        using var csvWriter = new CsvWriter(stringWriter, CultureInfo.InvariantCulture);
-        await csvWriter.WriteRecordsAsync(boreholes).ConfigureAwait(false);
-
-        return File(Encoding.UTF8.GetBytes(stringWriter.ToString()), "text/csv", "boreholes_export.csv");
-    }
-
     /// <summary>
     /// Asynchronously copies a <see cref="Borehole"/>.
     /// </summary>
@@ -312,7 +140,7 @@ public async Task<ActionResult<int>> CopyAsync([Required] int id, [Required] int
             return Unauthorized();
         }
 
-        var borehole = await GetBoreholesWithIncludes()
+        var borehole = await Context.Boreholes.GetAllWithIncludes()
             .AsNoTracking()
             .SingleOrDefaultAsync(b => b.Id == id)
             .ConfigureAwait(false);
@@ -424,33 +252,4 @@ public async Task<ActionResult<int>> CopyAsync([Required] int id, [Required] int
         if (entity == null) return default;
         return await Task.FromResult<int?>(entity.Id).ConfigureAwait(false);
     }
-
-    private IQueryable<Borehole> GetBoreholesWithIncludes()
-    {
-        return Context.Boreholes.Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerColorCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerDebrisCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerGrainAngularityCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerGrainShapeCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerOrganicComponentCodes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.Layers).ThenInclude(l => l.LayerUscs3Codes)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.LithologicalDescriptions)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.FaciesDescriptions)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.ChronostratigraphyLayers)
-            .Include(b => b.Stratigraphies).ThenInclude(s => s.LithostratigraphyLayers)
-            .Include(b => b.Completions).ThenInclude(c => c.Casings).ThenInclude(c => c.CasingElements)
-            .Include(b => b.Completions).ThenInclude(c => c.Instrumentations)
-            .Include(b => b.Completions).ThenInclude(c => c.Backfills)
-            .Include(b => b.Sections).ThenInclude(s => s.SectionElements)
-            .Include(b => b.Observations).ThenInclude(o => (o as FieldMeasurement)!.FieldMeasurementResults)
-            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestResults)
-            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestEvaluationMethodCodes)
-            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestFlowDirectionCodes)
-            .Include(b => b.Observations).ThenInclude(o => (o as Hydrotest)!.HydrotestKindCodes)
-            .Include(b => b.BoreholeCodelists)
-            .Include(b => b.Workflows)
-            .Include(b => b.BoreholeFiles)
-            .Include(b => b.BoreholeGeometry)
-            .Include(b => b.Workgroup)
-            .Include(b => b.UpdatedBy);
-    }
 }