test: improve loadtest maintenance

Use a northwind database extract, remove unnecessary loadtests.

Only one test using all the csv options in a single loadtest is necessary to
ensure perf doesn't drop when adding new options.

This makes the CI summary show fully instead of hiding some parts.

Author: steve-chavez

.github/workflows/ci.yaml

@@ -37,7 +37,7 @@ jobs:
   loadtest:
     strategy:
       matrix:
-        kind: ['csv_agg', 'csv_agg_delim', 'csv_agg_delim_bom_nullstr', 'postgrest']
+        kind: ['postgrest', 'csv_agg', 'csv_agg_options']
     name: Loadtest
     runs-on: ubuntu-24.04
     steps:

bench/csv_agg.sql

@@ -1,5 +1,5 @@
 \set lim random(1000, 2000)
 
 select csv_agg(t) from (
-  select * from student_emotion_assessments limit :lim
+  select * from orders_customers limit :lim
 ) as t;

bench/csv_agg_delim.sql

@@ -0,0 +1,5 @@
+\set lim random(1000, 2000)
+
+select csv_agg(t, csv_options(delimiter:='|', bom:=true, header:=false, nullstr:='<NULL>')) from (
+  select * from orders_customers limit :lim
+) as t;

bench/csv_agg_delim_bom.sql

@@ -1,130 +1,129 @@
-create extension if not exists pg_csv;
-
-create type gender_enum          as enum ('female', 'male', 'non_binary', 'prefer_not_to_say');
-create type attachment_enum      as enum ('secure', 'anxious', 'avoidant', 'fearful');
-create type regulation_strategy  as enum ('cognitive_reappraisal', 'suppression', 'rumination',
-                                          'problem_solving', 'distraction', 'other');
-
-create table student_emotion_assessments (
-    -- identifiers
-    assessment_id        bigserial primary key,
-    student_uuid         uuid        not null,
-    institution_id       int         not null,
-
-    -- demographics
-    gender               gender_enum not null,
-    birth_date           date        not null,
-    nationality          text        not null,
-    socioeconomic_level  text        not null,
-
-    -- academic context
-    faculty              text         not null,
-    degree_program       text         not null,
-    year_of_study        smallint     not null check (year_of_study between 1 and 7),
-    current_gpa          numeric(3,2) not null check (current_gpa between 0 and 4),
-    credits_completed    int          not null check (credits_completed >= 0),
-    enrollment_status    boolean      not null default true, -- true = active student
-
-    -- attachment style
-    attachment_style     attachment_enum not null,
-    attachment_score_anxiety  numeric(4,2) not null check (attachment_score_anxiety  between 1 and 7),
-    attachment_score_avoidant numeric(4,2) not null check (attachment_score_avoidant between 1 and 7),
-
-    -- difficulties in emotion regulation scale (ders-18) sub-scores
-    ders_non_acceptance  smallint not null check (ders_non_acceptance between 6 and 30),
-    ders_goals           smallint not null check (ders_goals between 5 and 25),
-    ders_impulse         smallint not null check (ders_impulse between 6 and 30),
-    ders_awareness       smallint not null check (ders_awareness between 6 and 30),
-    ders_strategy        smallint not null check (ders_strategy between 8 and 40),
-    ders_clarity         smallint not null check (ders_clarity between 5 and 25),
-    ders_total           smallint generated always as
-                         (ders_non_acceptance + ders_goals + ders_impulse +
-                          ders_awareness + ders_strategy + ders_clarity) stored,
-
-    -- emotion-regulation strategy prevalence (likert 1-5)
-    uses_reappraisal     smallint not null check (uses_reappraisal between 1 and 5),
-    uses_suppression     smallint not null check (uses_suppression between 1 and 5),
-    uses_rumination      smallint not null check (uses_rumination between 1 and 5),
-    predominant_strategy regulation_strategy not null,
+-- based on the northwind database https://github.com/pthom/northwind_psql
+-- the idea is to use the aggregate over a relation with lots of columns to test the performance
 
-    -- well-being & mental-health screeners
-    perceived_stress     smallint not null check (perceived_stress between 0 and 40),
-    anxiety_score_gad7   smallint not null check (anxiety_score_gad7 between 0 and 21),
-    depression_score_phq9 smallint not null check (depression_score_phq9 between 0 and 27),
+create extension if not exists pg_csv;
 
-    -- environmental variables
-    living_with_family   boolean not null,
-    weekly_work_hours    smallint not null check (weekly_work_hours between 0 and 60),
-    social_support_index smallint not null check (social_support_index between 12 and 84),
+CREATE TABLE customers (
+  customer_id   CHAR(5) PRIMARY KEY,
+  company_name  TEXT      NOT NULL,
+  contact_name  TEXT,
+  contact_title TEXT,
+  address       TEXT,
+  city          TEXT,
+  region        TEXT,
+  postal_code   TEXT,
+  country       TEXT,
+  phone         TEXT,
+  fax           TEXT
+);
 
-    -- audit fields
-    administered_by      text  not null, -- name/id of interviewer or system
-    collected_at         timestamptz  not null default now(),
-    updated_at           timestamptz  not null default now(),
-    constraint updated_at_future check (updated_at <= now())
+CREATE TABLE orders (
+  order_id         BIGSERIAL PRIMARY KEY,
+  customer_id      CHAR(5) NOT NULL REFERENCES customers(customer_id) ON DELETE CASCADE,
+  employee_id      SMALLINT,
+  order_date       DATE,
+  required_date    DATE,
+  shipped_date     DATE,
+  freight          NUMERIC(10,2) DEFAULT 0 CHECK (freight >= 0),
+  ship_name        TEXT,
+  ship_address     TEXT,
+  ship_city        TEXT,
+  ship_region      TEXT,
+  ship_postal_code TEXT,
+  ship_country     TEXT
 );
 
-INSERT INTO student_emotion_assessments (
-    student_uuid, institution_id, gender, birth_date, nationality, socioeconomic_level,
-    faculty, degree_program, year_of_study, current_gpa, credits_completed, enrollment_status,
-    attachment_style, attachment_score_anxiety, attachment_score_avoidant,
-    ders_non_acceptance, ders_goals, ders_impulse, ders_awareness, ders_strategy, ders_clarity,
-    uses_reappraisal, uses_suppression, uses_rumination, predominant_strategy,
-    perceived_stress, anxiety_score_gad7, depression_score_phq9,
-    living_with_family, weekly_work_hours, social_support_index,
-    administered_by
+-- generate seed data
+-- three groups of 100 by city/country
+INSERT INTO customers (
+  customer_id, company_name, contact_name, contact_title,
+  address, city, region, postal_code, country, phone, fax
+)
+SELECT
+  ('C' || lpad(i::text, 4, '0'))::char(5)                                    AS customer_id,
+  'Company ' || i                                                            AS company_name,
+  'Contact ' || i                                                            AS contact_name,
+  CASE
+    WHEN i <= 100 THEN 'Owner'
+    WHEN i <= 200 THEN 'Sales Manager'
+    ELSE 'Purchasing'
+  END                                                                        AS contact_title,
+  i::text || ' Main Street'                                                  AS address,
+  CASE
+    WHEN i <= 100 THEN 'Seattle'
+    WHEN i <= 200 THEN 'London'
+    ELSE 'Sao Paulo'
+  END                                                                        AS city,
+  CASE
+    WHEN i <= 100 THEN 'WA'
+    WHEN i <= 200 THEN NULL
+    ELSE 'SP'
+  END                                                                        AS region,
+  (10000 + i)::text                                                          AS postal_code,
+  CASE
+    WHEN i <= 100 THEN 'USA'
+    WHEN i <= 200 THEN 'UK'
+    ELSE 'Brazil'
+  END                                                                        AS country,
+  '+1-555-' || lpad(i::text, 4, '0')                                         AS phone,
+  CASE
+    WHEN right(i::text, 1) IN ('0','5') THEN NULL
+    ELSE '+1-555-' || lpad((i + 1000)::text, 4, '0')
+  END                                                                        AS fax
+FROM generate_series(1, 300) AS s(i);
+
+-- 2700 orders, 9 orders per customer
+WITH base AS (
+  SELECT c.customer_id, c.company_name, c.address, c.city, c.region, c.postal_code, c.country
+  FROM customers c
+)
+INSERT INTO orders (
+  customer_id, employee_id, order_date, required_date, shipped_date,
+  freight, ship_name, ship_address, ship_city, ship_region, ship_postal_code, ship_country
 )
 SELECT
-    gen_random_uuid(),                                     -- student_uuid
-    1 + (i % 5),                                           -- institution_id 1-5
-    CASE (i % 4)
-         WHEN 0 THEN 'female'
-         WHEN 1 THEN 'male'
-         WHEN 2 THEN 'non_binary'
-         ELSE 'prefer_not_to_say'
-    END::gender_enum,
-    (CURRENT_DATE - ((18 + (i % 10)) * INTERVAL '1 year'))::date,
-    'Country ' || i,
-    CASE WHEN i % 3 = 0 THEN 'alto'
-         WHEN i % 3 = 1 THEN 'medio'
-         ELSE 'bajo'
-    END,
-    CASE WHEN i % 2 = 0 THEN 'Psychology' ELSE 'Engineering' END,
-    CASE WHEN i % 2 = 0 THEN 'BSc' ELSE 'BA' END,
-    (i % 7) + 1,
-    round((random()*4)::numeric, 2)::numeric(3,2),         -- GPA 0-4.00
-    (i * 10) % 200,
-    TRUE,
-    CASE (i % 4)
-         WHEN 0 THEN 'secure'
-         WHEN 1 THEN 'anxious'
-         WHEN 2 THEN 'avoidant'
-         ELSE 'fearful'
-    END::attachment_enum,
-    3 + (i % 4),
-    3 + ((i + 2) % 4),
-    10 +  (i       % 15),     -- ders_non_acceptance
-    10 + ((i+1) % 15),        -- ders_goals
-    10 + ((i+2) % 15),        -- ders_impulse
-    10 + ((i+3) % 15),        -- ders_awareness
-    15 +  (i      % 10),      -- ders_strategy
-    5  + ((i+4) % 21),        -- ders_clarity  (range 5-25)
-    1 + (i % 5),              -- uses_reappraisal
-    1 + ((i+1) % 4),          -- uses_suppression
-    1 + ((i+2) % 3),          -- uses_rumination
-    CASE (i % 6)
-         WHEN 0 THEN 'cognitive_reappraisal'
-         WHEN 1 THEN 'suppression'
-         WHEN 2 THEN 'rumination'
-         WHEN 3 THEN 'problem_solving'
-         WHEN 4 THEN 'distraction'
-         ELSE 'other'
-    END::regulation_strategy,
-    10 + (i % 20),
-    2  + (i % 5),
-    3  + (i % 9),
-    (i % 2 = 0),
-    (i % 61),
-    20 + (i % 50),
-    'seed_script'
-FROM generate_series(1, 3000) AS s(i);
+  b.customer_id,
+  n::smallint                                                        AS employee_id,
+  (DATE '2024-01-01' + (n || ' day')::interval)::date               AS order_date,
+  (DATE '2024-01-01' + ((n + 7) || ' day')::interval)::date         AS required_date,
+  CASE WHEN n = 9 THEN NULL
+       ELSE (DATE '2024-01-01' + ((n + 3) || ' day')::interval)::date
+  END                                                               AS shipped_date,
+  (10 + n)::numeric(10,2)                                           AS freight,
+  b.company_name                                                    AS ship_name,
+  b.address                                                         AS ship_address,
+  b.city                                                            AS ship_city,
+  b.region                                                          AS ship_region,
+  b.postal_code                                                     AS ship_postal_code,
+  b.country                                                         AS ship_country
+FROM base b
+CROSS JOIN generate_series(1, 9) AS n;
+
+-- create a view to have more columns
+CREATE OR REPLACE VIEW orders_customers AS
+SELECT
+  o.order_id,
+  o.customer_id,
+  c.company_name,
+  c.contact_name,
+  c.contact_title,
+  c.address          AS customer_address,
+  c.city             AS customer_city,
+  c.region           AS customer_region,
+  c.postal_code      AS customer_postal_code,
+  c.country          AS customer_country,
+  c.phone,
+  c.fax,
+  o.employee_id,
+  o.order_date,
+  o.required_date,
+  o.shipped_date,
+  o.freight,
+  o.ship_name,
+  o.ship_address,
+  o.ship_city,
+  o.ship_region,
+  o.ship_postal_code,
+  o.ship_country
+FROM orders o
+JOIN customers c USING (customer_id);

bench/csv_agg_delim_bom_nullstr.sql

@@ -1,7 +1,7 @@
 \set lim random(1000, 2000)
 
 with pgrst_source as (
-  select * from student_emotion_assessments
+  select * from orders_customers limit :lim
 )
 select
     (select coalesce(string_agg(a.k, ','), '')  from (select json_object_keys(r)::text as k from (select row_to_json(hh) as r from pgrst_source as hh limit 1) _) a) ||