Merge pull request #4159 from dfe-analytical-services/EES-4389-api-an…

…d-next-caching-for-11-july

Ees 4389 api and next caching for 11 july

src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Cache/MemoryCacheAttributeTests.cs

@@ -10,6 +10,7 @@
 using Moq;
 using NCrontab;
 using Xunit;
+using static GovUk.Education.ExploreEducationStatistics.Common.Cache.CronSchedules;
 using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils;
 using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils;
 
@@ -18,9 +19,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Cache;
 [Collection(CacheTestFixture.CollectionName)]
 public class MemoryCacheAttributeTests : IClassFixture<CacheTestFixture>, IDisposable
 {
-    private const string HourlyExpirySchedule = "0 * * * *";
-    private const string HalfHourlyExpirySchedule = "*/30 * * * *";
-
     private readonly Mock<IMemoryCacheService> _memoryCacheService = new(MockBehavior.Strict);
 
     public MemoryCacheAttributeTests()

src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Services/MemoryCacheServiceTests.cs

@@ -11,15 +11,16 @@
 using NCrontab;
 using Xunit;
 using static System.Globalization.DateTimeStyles;
+using static GovUk.Education.ExploreEducationStatistics.Common.Cache.CronSchedules;
 using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils;
 using static Moq.MockBehavior;
 
 namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Services;
 
 public class MemoryCacheServiceTests
 {
-    private readonly CrontabSchedule _hourlyExpirySchedule = CrontabSchedule.Parse("0 * * * *");
-    private readonly CrontabSchedule _halfHourlyExpirySchedule = CrontabSchedule.Parse("*/30 * * * *");
+    private readonly CrontabSchedule _hourlyExpirySchedule = CrontabSchedule.Parse(HourlyExpirySchedule);
+    private readonly CrontabSchedule _halfHourlyExpirySchedule = CrontabSchedule.Parse(HalfHourlyExpirySchedule);
 
     private record SampleClassSuperclass;
 

src/GovUk.Education.ExploreEducationStatistics.Common/Cache/CronSchedules.cs

@@ -0,0 +1,7 @@
+namespace GovUk.Education.ExploreEducationStatistics.Common.Cache;
+
+public static class CronSchedules
+{
+    public const string HourlyExpirySchedule = "0 * * * *";
+    public const string HalfHourlyExpirySchedule = "*/30 * * * *";
+}

src/GovUk.Education.ExploreEducationStatistics.Common/ViewModels/PaginatedListViewModel.cs

@@ -4,6 +4,7 @@
 using System.Linq;
 using GovUk.Education.ExploreEducationStatistics.Common.Model;
 using Microsoft.AspNetCore.Mvc;
+using Newtonsoft.Json;
 
 namespace GovUk.Education.ExploreEducationStatistics.Common.ViewModels;
 
@@ -19,6 +20,13 @@ public PaginatedListViewModel(List<T> results, int totalResults, int page, int p
         Paging = new PagingViewModel(page: page, pageSize: pageSize, totalResults: totalResults);
     }
 
+    [JsonConstructor]
+    public PaginatedListViewModel(List<T> results, PagingViewModel paging)
+    {
+        Results = results;
+        Paging = paging;
+    }
+
     /// <summary>
     /// Paginate some results (in memory) that have not been paginated yet.
     /// </summary>

src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/PublicationControllerCachingTests.cs

@@ -0,0 +1,130 @@
+#nullable enable
+using System;
+using System.Threading.Tasks;
+using GovUk.Education.ExploreEducationStatistics.Common.Cache;
+using GovUk.Education.ExploreEducationStatistics.Common.Model;
+using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions;
+using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures;
+using GovUk.Education.ExploreEducationStatistics.Common.ViewModels;
+using GovUk.Education.ExploreEducationStatistics.Content.Api.Cache;
+using GovUk.Education.ExploreEducationStatistics.Content.Api.Controllers;
+using GovUk.Education.ExploreEducationStatistics.Content.Api.Requests;
+using GovUk.Education.ExploreEducationStatistics.Content.Model;
+using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces;
+using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.Cache;
+using GovUk.Education.ExploreEducationStatistics.Content.Services.ViewModels;
+using Moq;
+using NCrontab;
+using Xunit;
+using static GovUk.Education.ExploreEducationStatistics.Common.Cache.CronSchedules;
+using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils;
+using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils;
+using static Moq.MockBehavior;
+using static Newtonsoft.Json.JsonConvert;
+
+namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.Controllers;
+
+[Collection(CacheServiceTests)]
+public class PublicationControllerCachingTests : CacheServiceTestFixture
+{
+    private readonly PublicationsListRequest _query = new(
+        ReleaseType.ExperimentalStatistics,
+        ThemeId: Guid.Empty,
+        Search: "",
+        IPublicationService.PublicationsSortBy.Published,
+        SortOrder.Asc,
+        Page: 1,
+        PageSize: 10
+    );
+
+    private readonly PaginatedListViewModel<PublicationSearchResultViewModel> _publications = new(
+        ListOf(new PublicationSearchResultViewModel
+        {
+            Id = Guid.NewGuid(),
+            Published = DateTime.UtcNow,
+            Rank = 4,
+            Slug = "slug",
+            Summary = "summary",
+            Theme = "theme",
+            Title = "title",
+            Type = ReleaseType.ExperimentalStatistics
+        }), 5, 1, 10);
+
+    [Fact]
+    public async Task ListPublications_NoCachedEntryExists()
+    {
+        var publicationService = new Mock<IPublicationService>(Strict);
+
+        MemoryCacheService
+            .Setup(s => s.GetItem(
+                new GetPublicationListCacheKey(_query),
+                typeof(PaginatedListViewModel<PublicationSearchResultViewModel>)))
+            .Returns(null);
+
+        var expectedCacheConfiguration = new MemoryCacheConfiguration(
+            10, CrontabSchedule.Parse(HalfHourlyExpirySchedule));
+
+        MemoryCacheService
+            .Setup(s => s.SetItem<object>(
+                new GetPublicationListCacheKey(_query),
+                _publications,
+                ItIs.DeepEqualTo(expectedCacheConfiguration),
+                null));
+
+        publicationService
+            .Setup(s => s.ListPublications(
+                _query.ReleaseType,
+                _query.ThemeId,
+                _query.Search,
+                _query.Sort,
+                _query.Order,
+                _query.Page,
+                _query.PageSize))
+            .ReturnsAsync(_publications);
+
+        var controller = BuildController(publicationService.Object);
+
+        var result = await controller.ListPublications(_query);
+
+        VerifyAllMocks(MemoryCacheService, publicationService);
+
+        result.AssertOkResult(_publications);
+    }
+
+    [Fact]
+    public async Task ListPublications_CachedEntryExists()
+    {
+        MemoryCacheService
+            .Setup(s => s.GetItem(
+                new GetPublicationListCacheKey(_query),
+                typeof(PaginatedListViewModel<PublicationSearchResultViewModel>)))
+            .Returns(_publications);
+
+        var controller = BuildController();
+
+        var result = await controller.ListPublications(_query);
+
+        VerifyAllMocks(MemoryCacheService);
+
+        result.AssertOkResult(_publications);
+    }
+
+    [Fact]
+    public void PublicationSearchResults_SerializeAndDeserialize()
+    {
+        var converted = DeserializeObject<PaginatedListViewModel<PublicationSearchResultViewModel>>(
+            SerializeObject(_publications));
+
+        converted.AssertDeepEqualTo(_publications);
+    }
+
+    private static PublicationController BuildController(
+        IPublicationService? publicationService = null
+    )
+    {
+        return new(
+            Mock.Of<IPublicationCacheService>(Strict),
+            publicationService ?? Mock.Of<IPublicationService>(Strict)
+        );
+    }
+}

src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/ReleaseControllerCachingTests.cs

@@ -13,6 +13,7 @@
 using Moq;
 using NCrontab;
 using Xunit;
+using static GovUk.Education.ExploreEducationStatistics.Common.Cache.CronSchedules;
 using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils;
 using static Moq.MockBehavior;
 
@@ -21,8 +22,6 @@ namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Tests.Controlle
 [Collection(CacheServiceTests)]
 public class ReleaseControllerCachingTests : CacheServiceTestFixture
 {
-    private const string HalfHourlyExpirySchedule = "*/30 * * * *";
-
     private const string PublicationSlug = "publication-a";
     private const string ReleaseSlug = "200";
 

src/GovUk.Education.ExploreEducationStatistics.Content.Api/Cache/GetPublicationListCacheKey.cs

@@ -0,0 +1,9 @@
+using GovUk.Education.ExploreEducationStatistics.Common.Cache.Interfaces;
+using GovUk.Education.ExploreEducationStatistics.Content.Api.Requests;
+
+namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Cache;
+
+public record GetPublicationListCacheKey(PublicationsListRequest PublicationQuery) : IMemoryCacheKey
+{
+    public string Key => $"{GetType().Name}:{PublicationQuery}";
+}

src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/PublicationController.cs

@@ -2,15 +2,18 @@
 using System.Collections.Generic;
 using System.Net.Mime;
 using System.Threading.Tasks;
+using GovUk.Education.ExploreEducationStatistics.Common.Cache;
 using GovUk.Education.ExploreEducationStatistics.Common.Extensions;
 using GovUk.Education.ExploreEducationStatistics.Common.Model;
 using GovUk.Education.ExploreEducationStatistics.Common.ViewModels;
+using GovUk.Education.ExploreEducationStatistics.Content.Api.Cache;
 using GovUk.Education.ExploreEducationStatistics.Content.Api.Requests;
 using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces;
 using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.Cache;
 using GovUk.Education.ExploreEducationStatistics.Content.Services.Requests;
 using GovUk.Education.ExploreEducationStatistics.Content.Services.ViewModels;
 using Microsoft.AspNetCore.Mvc;
+using static GovUk.Education.ExploreEducationStatistics.Common.Cache.CronSchedules;
 
 namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Controllers
 {
@@ -45,6 +48,7 @@ public async Task<ActionResult<IList<PublicationTreeThemeViewModel>>> GetPublica
                 .HandleFailuresOrOk();
         }
 
+        [MemoryCache(typeof(GetPublicationListCacheKey), durationInSeconds: 10, expiryScheduleCron: HalfHourlyExpirySchedule)]
         [HttpGet("publications")]
         public async Task<ActionResult<PaginatedListViewModel<PublicationSearchResultViewModel>>> ListPublications(
             [FromQuery] PublicationsListRequest request)

src/GovUk.Education.ExploreEducationStatistics.Content.Api/Controllers/ReleaseController.cs

@@ -11,15 +11,14 @@
 using GovUk.Education.ExploreEducationStatistics.Content.Services.Interfaces.Cache;
 using GovUk.Education.ExploreEducationStatistics.Content.Services.ViewModels;
 using Microsoft.AspNetCore.Mvc;
+using static GovUk.Education.ExploreEducationStatistics.Common.Cache.CronSchedules;
 
 namespace GovUk.Education.ExploreEducationStatistics.Content.Api.Controllers
 {
     [Route("api")]
     [Produces(MediaTypeNames.Application.Json)]
     public class ReleaseController : ControllerBase
     {
-        private const string HalfHourlyExpirySchedule = "*/30 * * * *";
-
         private readonly IMethodologyCacheService _methodologyCacheService;
         private readonly IPublicationCacheService _publicationCacheService;
         private readonly IReleaseCacheService _releaseCacheService;

src/explore-education-statistics-admin/src/pages/release/datablocks/components/chart/ChartReferenceLinesConfiguration.tsx

@@ -6,7 +6,6 @@ import FormFieldSelect from '@common/components/form/FormFieldSelect';
 import FormFieldTextInput from '@common/components/form/FormFieldTextInput';
 import FormSelect, { SelectOption } from '@common/components/form/FormSelect';
 import Tooltip from '@common/components/Tooltip';
-import VisuallyHidden from '@common/components/VisuallyHidden';
 import {
   AxisType,
   ChartDefinitionAxis,

src/explore-education-statistics-frontend/.env

@@ -2,4 +2,5 @@ CONTENT_API_BASE_URL=http://localhost:5010/api
 DATA_API_BASE_URL=http://localhost:5000/api
 NOTIFICATION_API_BASE_URL=http://localhost:7073/api
 GA_TRACKING_ID=
-PUBLIC_URL=http://localhost:3000/
+PUBLIC_URL=http://localhost:3000/
+APP_ENV=Local

src/explore-education-statistics-frontend/server.js

@@ -13,6 +13,7 @@ if (process.env.APPINSIGHTS_INSTRUMENTATIONKEY) {
     .setAutoCollectDependencies(true)
     .setAutoCollectConsole(true)
     .setUseDiskRetryCaching(true)
+    .setSendLiveMetrics(true)
     .start();
 }
 

src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx

@@ -29,7 +29,7 @@ import { logEvent } from '@frontend/services/googleAnalyticsService';
 import glossaryService from '@frontend/services/glossaryService';
 import classNames from 'classnames';
 import orderBy from 'lodash/orderBy';
-import { GetServerSideProps, NextPage } from 'next';
+import { GetStaticPaths, GetStaticProps, NextPage } from 'next';
 import React from 'react';
 import VisuallyHidden from '@common/components/VisuallyHidden';
 import ScrollableContainer from '@common/components/ScrollableContainer';
@@ -571,13 +571,11 @@ const PublicationReleasePage: NextPage<Props> = ({ release }) => {
   );
 };
 
-export const getServerSideProps: GetServerSideProps<Props> = async ({
-  query,
-}) => {
+export const getStaticProps: GetStaticProps<Props> = async ({ params }) => {
   const {
     publication: publicationSlug,
     release: releaseSlug,
-  } = query as Dictionary<string>;
+  } = params as Dictionary<string>;
 
   const release = await (releaseSlug
     ? publicationService.getPublicationRelease(publicationSlug, releaseSlug)
@@ -587,6 +585,14 @@ export const getServerSideProps: GetServerSideProps<Props> = async ({
     props: {
       release,
     },
+    revalidate: process.env.APP_ENV === 'Local' ? 2 : 10,
+  };
+};
+
+export const getStaticPaths: GetStaticPaths = async () => {
+  return {
+    paths: [],
+    fallback: 'blocking',
   };
 };
 

src/explore-education-statistics-frontend/src/pages/find-statistics/[publication].tsx

@@ -1,4 +1,5 @@
 export {
   default,
-  getServerSideProps,
+  getStaticProps,
+  getStaticPaths,
 } from '@frontend/modules/find-statistics/PublicationReleasePage';

src/explore-education-statistics-frontend/src/pages/find-statistics/[publication]/[release].tsx

@@ -1,4 +1,5 @@
 export {
   default,
-  getServerSideProps,
+  getStaticProps,
+  getStaticPaths,
 } from '@frontend/modules/find-statistics/PublicationReleasePage';

src/explore-education-statistics-frontend/src/types/env.d.ts

@@ -1,6 +1,6 @@
 declare namespace NodeJS {
   interface ProcessEnv {
-    APP_ENV: 'Production' | 'Pre-Production' | 'Test' | 'Development';
+    APP_ENV: 'Production' | 'Pre-Production' | 'Test' | 'Development' | 'Local';
     APPINSIGHTS_INSTRUMENTATIONKEY: string;
     BUILD_NUMBER: string;
     GA_TRACKING_ID: string;

tests/robot-tests/.env.example

@@ -11,7 +11,8 @@ ANALYST_PASSWORD=ANALYST_PASSWORD  # Analyst1 user password
 EXPIRED_INVITE_USER_EMAIL= # Expired Invite user email
 EXPIRED_INVITE_USER_PASSWORD= # Expired Invite user password
 RELEASE_COMPLETE_WAIT=120  # Time to wait for a release to publish (in seconds)
-WAIT_MEMORY_CACHE_EXPIRY=2  # Time to wait for the memory cache to expire (in minutes. Local = 2, Dev = 10)
+WAIT_CACHE_EXPIRY=2  # Time to wait for the environment's various caches to expire (in seconds. Local = 2, Dev = 10)
+WAIT_NEXT_ISR=5  # Time for Next.js to perform Incremental Static Regeneration (generate a new version of a stale page in the background)
 WAIT_MEDIUM=120  # Variable used throughout tests (in seconds)
 WAIT_SMALL=45 # Variable used throughout tests (in seconds)
 WAIT_LONG=180  # Variable used throughout tests (in seconds)

tests/robot-tests/run_tests.py

@@ -178,7 +178,8 @@
     "WAIT_SMALL",
     "FAIL_TEST_SUITES_FAST",
     "IDENTITY_PROVIDER",
-    "WAIT_MEMORY_CACHE_EXPIRY",
+    "WAIT_CACHE_EXPIRY",
+    "WAIT_NEXT_ISR",
     "EXPIRED_INVITE_USER_EMAIL",
     "PUBLISHER_FUNCTIONS_URL",
 ]