Merge pull request #29 from xforman2/28-more-effective-data-fetching

28 more effective data fetching
xforman2 · Sep 3, 2024 · 9eed5e6 · 9eed5e6
2 parents 406067c + f80c2d4
commit 9eed5e6
Show file tree

Hide file tree

Showing 6 changed files with 216 additions and 4 deletions.
diff --git a/Backend/Backend.csproj b/Backend/Backend.csproj
@@ -14,12 +14,19 @@
       <None Update="appsettings.json">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       </None>
+      <None Update="appsettings.json">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
     </ItemGroup>
 
     <ItemGroup>
       <PackageReference Include="Bogus" Version="35.6.0" />
       <PackageReference Include="GraphQL.Client" Version="6.1.0" />
       <PackageReference Include="GraphQL.Client.Serializer.Newtonsoft" Version="6.1.0" />
+      <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.8">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      </PackageReference>
       <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
       <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />

diff --git a/Backend/Program.cs b/Backend/Program.cs
@@ -18,7 +18,6 @@ static async Task Main(string[] args)
         // Main scope of application
         using (var scope = host.Services.CreateScope())
         {
-            //TODO this is just temp, delete later
             var services = scope.ServiceProvider;
 
             var dataProcessingService = services.GetRequiredService<DataProcessingService>();

diff --git a/Backend/Services/DataProcessingService.cs b/Backend/Services/DataProcessingService.cs
@@ -1,3 +1,4 @@
+using System.Diagnostics;
 using Backend.Clients;
 using Backend.Mappers;
 using HouseScout.Model;
@@ -21,15 +22,39 @@ HouseScoutContext context
 
     public async Task ProcessData()
     {
+        var existingEstates = _context.Estates.ToList();
+
+        var fetchedEstates = new List<Estate>();
+
         foreach (var kvp in _clientsAndMappers)
         {
             IClient client = kvp.Key;
             IMapper mapper = kvp.Value;
 
             var data = await client.FetchDataAsync();
-            var mappedData = mapper.MapResponseToModel(data);
-            _context.Estates.AddRange(mappedData);
-            await _context.SaveChangesAsync();
+            fetchedEstates.AddRange(mapper.MapResponseToModel(data));
+        }
+
+        var fetchedEstatesSet = new HashSet<string>(fetchedEstates.Select(fe => fe.ApiId));
+        var existingEstatesSet = new HashSet<string>(existingEstates.Select(fe => fe.ApiId));
+
+        var estatesToAdd = fetchedEstates
+            .Where(fe => !existingEstatesSet.Contains(fe.ApiId))
+            .ToList();
+
+        var estatesToRemove = existingEstates
+            .Where(ee => !fetchedEstatesSet.Contains(ee.ApiId))
+            .ToList();
+
+        if (estatesToAdd.Any())
+        {
+            await _context.Estates.AddRangeAsync(estatesToAdd);
+        }
+
+        if (estatesToRemove.Any())
+        {
+            _context.Estates.RemoveRange(estatesToRemove);
         }
+        await _context.SaveChangesAsync();
     }
 }
diff --git a/SharedDependencies/Migrations/20240901114150_InitialCreate.Designer.cs b/SharedDependencies/Migrations/20240901114150_InitialCreate.Designer.cs
diff --git a/SharedDependencies/Migrations/20240901114150_InitialCreate.cs b/SharedDependencies/Migrations/20240901114150_InitialCreate.cs
@@ -0,0 +1,46 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace SharedDependencies.Migrations
+{
+    /// <inheritdoc />
+    public partial class InitialCreate : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "Estates",
+                columns: table => new
+                {
+                    Id = table
+                        .Column<int>(type: "integer", nullable: false)
+                        .Annotation(
+                            "Npgsql:ValueGenerationStrategy",
+                            NpgsqlValueGenerationStrategy.IdentityByDefaultColumn
+                        ),
+                    ApiType = table.Column<int>(type: "integer", nullable: false),
+                    ApiId = table.Column<string>(type: "text", nullable: false),
+                    Address = table.Column<string>(type: "text", nullable: false),
+                    Price = table.Column<decimal>(type: "numeric", nullable: false),
+                    Link = table.Column<string>(type: "text", nullable: false),
+                    Surface = table.Column<double>(type: "double precision", nullable: false),
+                    EstateType = table.Column<int>(type: "integer", nullable: false),
+                    OfferType = table.Column<int>(type: "integer", nullable: false),
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_Estates", x => x.Id);
+                }
+            );
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(name: "Estates");
+        }
+    }
+}
diff --git a/SharedDependencies/Migrations/HouseScoutContextModelSnapshot.cs b/SharedDependencies/Migrations/HouseScoutContextModelSnapshot.cs
@@ -0,0 +1,66 @@
+// <auto-generated />
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+using SharedDependencies.Model;
+
+#nullable disable
+
+namespace SharedDependencies.Migrations
+{
+    [DbContext(typeof(HouseScoutContext))]
+    partial class HouseScoutContextModelSnapshot : ModelSnapshot
+    {
+        protected override void BuildModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "8.0.8")
+                .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+            NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+            modelBuilder.Entity("HouseScout.Model.Estate", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<string>("Address")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.Property<string>("ApiId")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.Property<int>("ApiType")
+                        .HasColumnType("integer");
+
+                    b.Property<int>("EstateType")
+                        .HasColumnType("integer");
+
+                    b.Property<string>("Link")
+                        .IsRequired()
+                        .HasColumnType("text");
+
+                    b.Property<int>("OfferType")
+                        .HasColumnType("integer");
+
+                    b.Property<decimal>("Price")
+                        .HasColumnType("numeric");
+
+                    b.Property<double>("Surface")
+                        .HasColumnType("double precision");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("Estates");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}