From c9cbddecd2adb3013282e473b01d37937c50d812 Mon Sep 17 00:00:00 2001 From: David Dudas Date: Tue, 18 Feb 2025 13:43:20 -0800 Subject: [PATCH 1/2] [Issue 2968] Add execution timers to ETL workflow logging (#3918) ## Summary Fixes #2968 ### Time to review: __2 mins__ ## Changes proposed > What was added, updated, or removed in this PR. Adds a few simple execution timers so we can easily measure and inspect how long the ETL workflow takes to execute. ## Context for reviewers > Testing instructions, background context, more in-depth details of the implementation, and anything else you'd like to call out or ask reviewers. Explain how the changes were verified. It was suggested that we optimize the ETL workflow insert pattern. A precursor to any optimization is measuring the baseline performance. This PR introduces simple execution timers to measure baseline performance. ## Additional information > Screenshots, GIF demos, code examples or output to help show the changes working as expected. --- analytics/src/analytics/cli.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/analytics/src/analytics/cli.py b/analytics/src/analytics/cli.py index d535b4d12..5d4479e5e 100644 --- a/analytics/src/analytics/cli.py +++ b/analytics/src/analytics/cli.py @@ -4,6 +4,7 @@ import logging import logging.config +import time from datetime import datetime from pathlib import Path from typing import Annotated @@ -82,8 +83,13 @@ def export_github_data( config = load_config(config_path, GitHubProjectConfig) config.temp_dir = temp_dir config.output_file = output_file + # Run ETL pipeline + logger.info("running extract workflow") + start_time = time.perf_counter() GitHubProjectETL(config).run() + end_time = time.perf_counter() + logger.info("extract workflow executed in %.5f seconds", end_time - start_time) # =========================================================== @@ -143,13 +149,12 @@ def transform_and_load( logger.info("running transform and load with effective date %s", datestamp) # hydrate a dataset instance from the input data + start_time = time.perf_counter() dataset = EtlDataset.load_from_json_file(file_path=issue_file) - # sync data to db etldb.sync_data(dataset, datestamp) - - # finish - logger.info("transform and load is done") + end_time = time.perf_counter() + logger.info("transform and load is done after %.5f seconds", end_time - start_time) @etl_app.command(name="extract_transform_and_load") @@ -175,16 +180,21 @@ def extract_transform_and_load( # extract data from GitHub logger.info("extracting data from GitHub") + start_time = time.perf_counter() extracted_json = GitHubProjectETL(config).extract_and_transform_in_memory() + end_time = time.perf_counter() + logger.info("extract executed in %.5f seconds", end_time - start_time) + # hydrate a dataset instance from the input data - logger.info("transforming data") + logger.info("transforming and loading data") + start_time = time.perf_counter() dataset = EtlDataset.load_from_json_object(json_data=extracted_json) - # sync dataset to db - logger.info("writing to database") etldb.sync_data(dataset, datestamp) + end_time = time.perf_counter() + logger.info("transform and load executed in %.5f seconds", end_time - start_time) - logger.info("workflow is done!") + logger.info("ETL workflow is done") def validate_effective_date(effective_date: str) -> str | None: From fa73e2d648618cbc7d5be8f268f5f820078f3ca3 Mon Sep 17 00:00:00 2001 From: KeithNava <134446588+KeithNava@users.noreply.github.com> Date: Wed, 19 Feb 2025 10:17:23 -0500 Subject: [PATCH 2/2] [Issue #2537] Create e2e tests for subscribe page (#3787) ## Summary Fixes #2537 ### Time to review: 15 mins ## Changes proposed Creates the E2E tests for the /subscribe page. --- .github/workflows/ci-frontend-e2e.yml | 2 + frontend/next.config.js | 3 ++ frontend/tests/e2e/subscribe.spec.ts | 69 +++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 frontend/tests/e2e/subscribe.spec.ts diff --git a/.github/workflows/ci-frontend-e2e.yml b/.github/workflows/ci-frontend-e2e.yml index a824d8e9d..17af0ca4a 100644 --- a/.github/workflows/ci-frontend-e2e.yml +++ b/.github/workflows/ci-frontend-e2e.yml @@ -59,6 +59,8 @@ jobs: run: | sed -En '/API_JWT_PUBLIC_KEY/,/-----END PUBLIC KEY-----/p' ../api/override.env >> .env.local cat .env.development >> .env.local + # Use the localhost url for tests + sed -i 's|^SENDY_API_URL=.*|SENDY_API_URL=http://localhost:3000|' .env.local npm run build -- --no-lint - name: Run e2e tests (Shard ${{ matrix.shard }}/${{ matrix.total_shards }}) diff --git a/frontend/next.config.js b/frontend/next.config.js index d87cf3b74..11df24df8 100644 --- a/frontend/next.config.js +++ b/frontend/next.config.js @@ -101,6 +101,9 @@ const nextConfig = { "types", ], }, + experimental: { + testProxy: true, + }, }; module.exports = withNextIntl(nextConfig); diff --git a/frontend/tests/e2e/subscribe.spec.ts b/frontend/tests/e2e/subscribe.spec.ts new file mode 100644 index 000000000..f51ad992c --- /dev/null +++ b/frontend/tests/e2e/subscribe.spec.ts @@ -0,0 +1,69 @@ +/* eslint-disable testing-library/prefer-screen-queries */ + +import { + expect, + NextFixture, + test, +} from "next/experimental/testmode/playwright"; + +function mockAPIEndpoints(next: NextFixture, responseText = "1") { + next.onFetch((request: Request) => { + if (request.url.endsWith("/subscribe") && request.method === "POST") { + return new Response(responseText, { + status: 200, + headers: { + "Content-Type": "text/plain", + }, + }); + } + + return "abort"; + }); +} +test.beforeEach(async ({ page }) => { + await page.goto("/subscribe"); +}); + +test.afterEach(async ({ context }) => { + await context.close(); +}); + +test("has title", async ({ page }) => { + await expect(page).toHaveTitle(/Subscribe | Simpler.Grants.gov/); +}); + +test("client side errors", async ({ page }) => { + await page.getByRole("button", { name: /subscribe/i }).click(); + + // Verify client-side errors for required fields + await expect(page.getByTestId("errorMessage")).toHaveCount(2); + await expect(page.getByText("Please enter a name.")).toBeVisible(); + await expect(page.getByText("Please enter an email address.")).toBeVisible(); +}); + +test("successful signup", async ({ next, page }) => { + mockAPIEndpoints(next); + + await page.getByLabel("First Name (required)").fill("Apple"); + await page.getByLabel("Email (required)").fill("name@example.com"); + + await page.getByRole("button", { name: /subscribe/i }).click(); + + await expect( + page.getByRole("heading", { name: /you['’]re subscribed/i }), + ).toBeVisible(); +}); + +test("error during signup", async ({ next, page }) => { + mockAPIEndpoints(next, "Error with subscribing"); + + await page.getByLabel("First Name (required)").fill("Apple"); + await page.getByLabel("Email (required)").fill("name@example.com"); + + await page.getByRole("button", { name: /subscribe/i }).click(); + + await expect(page.getByTestId("errorMessage")).toHaveCount(1); + await expect( + page.getByText(/an error occurred when trying to save your subscription./i), + ).toBeVisible(); +});