diff --git a/internal/impl/oracledb/bench/Taskfile.yaml b/internal/impl/oracledb/bench/Taskfile.yaml new file mode 100644 index 0000000000..b91c4567b3 --- /dev/null +++ b/internal/impl/oracledb/bench/Taskfile.yaml @@ -0,0 +1,53 @@ +version: '3' + +# Running order: +# - task oracledb:up +# - task oracledb:archivelog +# - task rman:setup +# - task sqlcl:create +# - task sqlcl:data:users + +tasks: + oracledb:up: + cmds: + - docker run -d + --name oracledb + -p 1521:1521 + -e ORACLE_PWD=YourPassword123 + container-registry.oracle.com/database/express:latest + # - task: oracledb:archivelog + # - task: rman:setup + + oracledb:down: + cmd: docker rm -fv oracledb + + oracledb:logs: + cmd: docker logs -f oracledb + + sqlcl: + cmd: sqlcl system/YourPassword123@localhost:1521/XE {{.EXTRA_ARGS}} + + sqlcl:create: + cmd: task sqlcl EXTRA_ARGS="@create.sql" + + sqlcl:data:users: + cmd: task sqlcl EXTRA_ARGS="@users.sql" + + sqlcl:data:products: + cmd: task sqlcl EXTRA_ARGS="@products.sql" + + sqlcl:data:cart: + cmd: task sqlcl EXTRA_ARGS="@cart.sql" + + sqlcl:drop-cache: + cmd: echo "DROP TABLE RPCN.CDC_CHECKPOINT_CACHE;" | sqlcl system/YourPassword123@localhost:1521/XE + + oracledb:archivelog: + desc: Enable ARCHIVELOG mode (required for LogMiner/CDC). Must be run after oracledb:up. + cmds: + - docker exec oracledb mkdir -p /opt/oracle/oradata/recovery_area + - docker exec -i oracledb sqlplus / as sysdba < archivelog_enable.sql + + rman:setup: + desc: Configure RMAN archive log retention policy for local CDC development + cmd: docker exec -i oracledb rman target / < rman_setup.rman diff --git a/internal/impl/oracledb/bench/archivelog_enable.sql b/internal/impl/oracledb/bench/archivelog_enable.sql new file mode 100644 index 0000000000..ed6c481a50 --- /dev/null +++ b/internal/impl/oracledb/bench/archivelog_enable.sql @@ -0,0 +1,11 @@ +SHUTDOWN ABORT; +STARTUP; +SHUTDOWN IMMEDIATE; +STARTUP MOUNT; +ALTER DATABASE ARCHIVELOG; +ALTER DATABASE OPEN; +ALTER PLUGGABLE DATABASE ALL OPEN; +ALTER SYSTEM SET db_recovery_file_dest_size = 10G SCOPE=BOTH; +ALTER SYSTEM SET db_recovery_file_dest = '/opt/oracle/oradata/recovery_area' SCOPE=BOTH; +SELECT LOG_MODE FROM V$DATABASE; +EXIT; diff --git a/internal/impl/oracledb/bench/benchmark_config.yaml b/internal/impl/oracledb/bench/benchmark_config.yaml new file mode 100644 index 0000000000..35111aa2bd --- /dev/null +++ b/internal/impl/oracledb/bench/benchmark_config.yaml @@ -0,0 +1,40 @@ +http: + debug_endpoints: true + +input: + oracledb_cdc: + connection_string: oracle://system:YourPassword123@localhost:1521/XE + stream_snapshot: false + snapshot_max_batch_size: 500 + logminer: + scn_window_size: 85000 + backoff_interval: 2s + mining_interval: 0s + include: + - TESTDB.USERS + - TESTDB.PRODUCTS + - TESTDB.CART + batching: + count: 30000 + period: 1s + + + +output: + processors: + - benchmark: + interval: 1s + count_bytes: true + file: + path: "./internal/impl/oracledb/bench/results.json" + codec: lines + # stdout: {} + # drop: {} + +logger: + level: INFO + +metrics: + prometheus: + add_process_metrics: true + add_go_metrics: true diff --git a/internal/impl/oracledb/bench/cart.sql b/internal/impl/oracledb/bench/cart.sql new file mode 100644 index 0000000000..4f1b2f65d8 --- /dev/null +++ b/internal/impl/oracledb/bench/cart.sql @@ -0,0 +1,67 @@ +-- Oracle Database Benchmark - Cart Data +-- Connection: oracle://system:YourPassword123@localhost:1521/XE +-- Prerequisites: Run create.sql first + +-- Enable output for debugging +SET SERVEROUTPUT ON; + +-- Switch to testdb schema +ALTER SESSION SET CURRENT_SCHEMA = testdb; +/ + +DECLARE + cart_total NUMBER := 10000000; + cart_batch_size NUMBER := 10000; + cart_current NUMBER := 0; + batch_end NUMBER; +BEGIN + DBMS_OUTPUT.PUT_LINE('Inserting test data into testdb.cart (' || cart_total || ' rows)...'); + + -- Oracle transactions start automatically, no explicit BEGIN needed + WHILE cart_current < cart_total + LOOP + batch_end := cart_current + cart_batch_size; + IF batch_end > cart_total THEN + batch_end := cart_total; + END IF; + + -- Insert batch using a CTE-style approach + INSERT INTO testdb.cart (name, email, info, date_of_birth, created_at, is_active, login_count, balance) + SELECT + 'cart-' || n, -- name + 'cart' || n || '@example.com', -- email + RPAD('This is about cart ' || n || '. ', 1000, 'X'), -- info (40 repetitions ~1KB) + SYSDATE - MOD(n, 10000), -- date_of_birth, spread over ~27 years + SYSTIMESTAMP, -- created_at + CASE WHEN MOD(n, 2) = 0 THEN 1 ELSE 0 END, -- is_active alternating 1/0 + MOD(n, 100), -- login_count between 0-99 + CAST(MOD(n, 1000) + MOD(n, 100) / 100.0 AS NUMBER(10,2)) -- balance + FROM ( + SELECT ROWNUM + cart_current AS n + FROM dual + CONNECT BY LEVEL <= (batch_end - cart_current) + ); + + cart_current := batch_end; + + -- Log progress after every batch + DBMS_OUTPUT.PUT_LINE('Progress: ' || cart_current || '/' || cart_total || ' rows inserted into testdb.cart'); + + -- Explicitly commit the current transaction + COMMIT; + + -- Oracle automatically starts a new transaction after COMMIT + END LOOP; + + DBMS_OUTPUT.PUT_LINE('Completed: ' || cart_current || ' rows inserted into testdb.cart'); +END; +/ + +-- Verification +DECLARE + cart_count NUMBER; +BEGIN + SELECT COUNT(*) INTO cart_count FROM testdb.cart; + DBMS_OUTPUT.PUT_LINE('Verification - testdb.cart: ' || cart_count || ' rows'); +END; +/ diff --git a/internal/impl/oracledb/bench/create.sql b/internal/impl/oracledb/bench/create.sql new file mode 100644 index 0000000000..4c068e2713 --- /dev/null +++ b/internal/impl/oracledb/bench/create.sql @@ -0,0 +1,232 @@ +-- Oracle Database Benchmark Setup Script +-- This script creates the user/schema, enables supplemental logging, and creates tables +-- Connection: oracle://system:YourPassword123@localhost:1521/XE + +-- Enable creation of local users in CDB root (not recommended for production) +ALTER SESSION SET "_ORACLE_SCRIPT"=TRUE; +/ + +-- ============================================================================ +-- STAGE 1: Create User/Schema +-- ============================================================================ +BEGIN + DBMS_OUTPUT.PUT_LINE('=== STAGE 1: Creating testdb user ==='); +END; +/ + +DECLARE + user_exists NUMBER; +BEGIN + SELECT COUNT(*) INTO user_exists FROM dba_users WHERE username = 'TESTDB'; + + IF user_exists = 0 THEN + EXECUTE IMMEDIATE 'CREATE USER testdb IDENTIFIED BY testdb123'; + EXECUTE IMMEDIATE 'GRANT CONNECT, RESOURCE, DBA TO testdb'; + EXECUTE IMMEDIATE 'GRANT UNLIMITED TABLESPACE TO testdb'; + EXECUTE IMMEDIATE 'ALTER SYSTEM SET ARCHIVE_LAG_TARGET = 60 SCOPE=BOTH'; + EXECUTE IMMEDIATE 'ALTER SYSTEM SET LOG_ARCHIVE_RETENTION_HOURS = 24;'; + DBMS_OUTPUT.PUT_LINE('User testdb created successfully'); + ELSE + DBMS_OUTPUT.PUT_LINE('User testdb already exists'); + END IF; +END; +/ + +-- ============================================================================ +-- STAGE 2: Enable Supplemental Logging for CDC +-- ============================================================================ +BEGIN + DBMS_OUTPUT.PUT_LINE('=== STAGE 2: Enabling supplemental logging ==='); +END; +/ + +-- Enable minimal supplemental logging at database level +ALTER DATABASE ADD SUPPLEMENTAL LOG DATA; + +-- Enable primary key and unique key supplemental logging +ALTER DATABASE ADD SUPPLEMENTAL LOG DATA (PRIMARY KEY, UNIQUE) COLUMNS; + +BEGIN + DBMS_OUTPUT.PUT_LINE('Supplemental logging enabled'); +END; +/ + +-- ============================================================================ +-- STAGE 3: Create Tables and Enable Supplemental Logging +-- ============================================================================ +BEGIN + DBMS_OUTPUT.PUT_LINE('=== STAGE 3: Creating tables and enabling CDC ==='); +END; +/ + +-- Switch to testdb user context +ALTER SESSION SET CURRENT_SCHEMA = testdb; +/ + +-- Create rpcn user if needed (Oracle uses users/schemas interchangeably) +DECLARE + user_exists NUMBER; +BEGIN + SELECT COUNT(*) INTO user_exists FROM dba_users WHERE username = 'RPCN'; + + IF user_exists = 0 THEN + EXECUTE IMMEDIATE 'CREATE USER rpcn IDENTIFIED BY rpcn123'; + EXECUTE IMMEDIATE 'GRANT CONNECT, RESOURCE TO rpcn'; + EXECUTE IMMEDIATE 'GRANT UNLIMITED TABLESPACE TO rpcn'; + DBMS_OUTPUT.PUT_LINE('User rpcn created'); + ELSE + DBMS_OUTPUT.PUT_LINE('User rpcn already exists'); + END IF; +END; +/ + +-- Create testdb.users table +BEGIN + DBMS_OUTPUT.PUT_LINE('Creating table testdb.users...'); +END; +/ + +DECLARE + table_exists NUMBER; +BEGIN + SELECT COUNT(*) INTO table_exists + FROM user_tables + WHERE table_name = 'USERS'; + + IF table_exists = 0 THEN + EXECUTE IMMEDIATE ' + CREATE TABLE testdb.users ( + id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name NVARCHAR2(100) NOT NULL, + surname NVARCHAR2(100) NOT NULL, + email NVARCHAR2(255) NOT NULL, + date_of_birth DATE, + join_date DATE, + created_at TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL, + is_active NUMBER(1) DEFAULT 1 NOT NULL, + login_count NUMBER DEFAULT 0 NOT NULL, + balance NUMBER(10,2) DEFAULT 0.00 NOT NULL + )'; + + -- Enable supplemental logging for this table + EXECUTE IMMEDIATE 'ALTER TABLE testdb.users ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS'; + + DBMS_OUTPUT.PUT_LINE('Table testdb.users created and supplemental logging enabled'); + ELSE + DBMS_OUTPUT.PUT_LINE('Table testdb.users already exists'); + END IF; +END; +/ + +-- Create testdb.products table +BEGIN + DBMS_OUTPUT.PUT_LINE('Creating table testdb.products...'); +END; +/ + +DECLARE + table_exists NUMBER; +BEGIN + SELECT COUNT(*) INTO table_exists + FROM user_tables + WHERE table_name = 'PRODUCTS'; + + IF table_exists = 0 THEN + EXECUTE IMMEDIATE ' + CREATE TABLE testdb.products ( + id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name NVARCHAR2(100) NOT NULL, + info NVARCHAR2(100) NOT NULL, + description NCLOB NOT NULL, + email NVARCHAR2(255) NOT NULL, + date_added DATE, + join_date DATE, + created_at TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL, + is_active NUMBER(1) DEFAULT 1 NOT NULL, + basket_count NUMBER DEFAULT 0 NOT NULL, + price NUMBER(10,2) DEFAULT 0.00 NOT NULL + )'; + + -- Enable supplemental logging for this table + EXECUTE IMMEDIATE 'ALTER TABLE testdb.products ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS'; + + DBMS_OUTPUT.PUT_LINE('Table testdb.products created and supplemental logging enabled'); + ELSE + DBMS_OUTPUT.PUT_LINE('Table testdb.products already exists'); + END IF; +END; +/ + +-- Create testdb.cart table +BEGIN + DBMS_OUTPUT.PUT_LINE('Creating table testdb.cart...'); +END; +/ + +DECLARE + table_exists NUMBER; +BEGIN + SELECT COUNT(*) INTO table_exists + FROM user_tables + WHERE table_name = 'CART'; + + IF table_exists = 0 THEN + EXECUTE IMMEDIATE ' + CREATE TABLE testdb.cart ( + id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name NVARCHAR2(100) NOT NULL, + info NCLOB NOT NULL, + email NVARCHAR2(255) NOT NULL, + date_of_birth DATE, + created_at TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL, + is_active NUMBER(1) DEFAULT 1 NOT NULL, + login_count NUMBER DEFAULT 0 NOT NULL, + balance NUMBER(10,2) DEFAULT 0.00 NOT NULL + )'; + + -- Enable supplemental logging for this table + EXECUTE IMMEDIATE 'ALTER TABLE testdb.cart ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS'; + + DBMS_OUTPUT.PUT_LINE('Table testdb.cart created and supplemental logging enabled'); + ELSE + DBMS_OUTPUT.PUT_LINE('Table testdb.cart already exists'); + END IF; +END; +/ + +-- Create testdb.cart2 table +BEGIN + DBMS_OUTPUT.PUT_LINE('Creating table testdb.cart2...'); +END; +/ + +DECLARE + table_exists NUMBER; +BEGIN + SELECT COUNT(*) INTO table_exists + FROM user_tables + WHERE table_name = 'CART2'; + + IF table_exists = 0 THEN + EXECUTE IMMEDIATE ' + CREATE TABLE testdb.cart2 ( + id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name NVARCHAR2(100) NOT NULL, + info NCLOB NOT NULL, + email NVARCHAR2(255) NOT NULL, + date_of_birth DATE, + created_at TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL, + is_active NUMBER(1) DEFAULT 1 NOT NULL, + login_count NUMBER DEFAULT 0 NOT NULL, + balance NUMBER(10,2) DEFAULT 0.00 NOT NULL + )'; + + -- Enable supplemental logging for this table + EXECUTE IMMEDIATE 'ALTER TABLE testdb.cart2 ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS'; + + DBMS_OUTPUT.PUT_LINE('Table testdb.cart2 created and supplemental logging enabled'); + ELSE + DBMS_OUTPUT.PUT_LINE('Table testdb.cart2 already exists'); + END IF; +END; +/ diff --git a/internal/impl/oracledb/bench/products.sql b/internal/impl/oracledb/bench/products.sql new file mode 100644 index 0000000000..5f0a93369b --- /dev/null +++ b/internal/impl/oracledb/bench/products.sql @@ -0,0 +1,65 @@ +-- Oracle Database Benchmark - Products Data +-- Connection: oracle://system:YourPassword123@localhost:1521/XE +-- Prerequisites: Run create.sql first + +-- Enable output for debugging +SET SERVEROUTPUT ON; + +-- Switch to testdb schema +ALTER SESSION SET CURRENT_SCHEMA = testdb; +/ + +DECLARE + products_total NUMBER := 500000; + products_batch_size NUMBER := 10000; + products_current NUMBER := 0; + products_batch_end NUMBER; +BEGIN + DBMS_OUTPUT.PUT_LINE('Inserting test data into testdb.products (' || products_total || ' rows)...'); + + WHILE products_current < products_total + LOOP + products_batch_end := products_current + products_batch_size; + IF products_batch_end > products_total THEN + products_batch_end := products_total; + END IF; + + -- Insert batch using a CTE-style approach + INSERT INTO testdb.products (name, info, description, email, date_added, join_date, created_at, is_active, basket_count, price) + SELECT + 'product-' || n, -- name + 'info-' || n, -- info + RPAD('This is about product ' || n || '. ', 500000, 'X'), -- description ~500 KB + 'help' || n || '@example.com', -- email + SYSDATE - MOD(n, 10000), -- date_added, spread over ~27 years + SYSTIMESTAMP, -- join_date + SYSTIMESTAMP, -- created_at + CASE WHEN MOD(n, 2) = 0 THEN 1 ELSE 0 END, -- is_active alternating 1/0 + MOD(n, 100), -- basket_count between 0-99 + CAST(MOD(n, 1000) + MOD(n, 100) / 100.0 AS NUMBER(10,2)) -- price + FROM ( + SELECT ROWNUM + products_current AS n + FROM dual + CONNECT BY LEVEL <= (products_batch_end - products_current) + ); + + COMMIT; + + products_current := products_batch_end; + + -- Log progress after every batch + DBMS_OUTPUT.PUT_LINE('Progress: ' || products_current || '/' || products_total || ' rows inserted into testdb.products'); + END LOOP; + + DBMS_OUTPUT.PUT_LINE('Completed: ' || products_current || ' rows inserted into testdb.products'); +END; +/ + +-- Verification +DECLARE + products_count NUMBER; +BEGIN + SELECT COUNT(*) INTO products_count FROM testdb.products; + DBMS_OUTPUT.PUT_LINE('Verification - testdb.products: ' || products_count || ' rows'); +END; +/ diff --git a/internal/impl/oracledb/bench/rman_setup.rman b/internal/impl/oracledb/bench/rman_setup.rman new file mode 100644 index 0000000000..b135bdb28d --- /dev/null +++ b/internal/impl/oracledb/bench/rman_setup.rman @@ -0,0 +1,9 @@ +# RMAN setup script for Oracle CDC local development +# Configures archive log retention so LogMiner has logs available to mine. +# Run via: task rman:setup + +# Keep archive logs needed to recover from any point in the last 24 hours. +# This prevents RMAN from marking logs as obsolete before LogMiner can read them. +CONFIGURE RETENTION POLICY TO RECOVERY WINDOW OF 1 DAYS; + +EXIT; diff --git a/internal/impl/oracledb/bench/users.sql b/internal/impl/oracledb/bench/users.sql new file mode 100644 index 0000000000..a78fb27ce4 --- /dev/null +++ b/internal/impl/oracledb/bench/users.sql @@ -0,0 +1,67 @@ +-- Oracle Database Benchmark - Users Data +-- Connection: oracle://system:YourPassword123@localhost:1521/XE +-- Prerequisites: Run create.sql first + +-- Enable output for debugging +SET SERVEROUTPUT ON; + +-- Switch to testdb schema +ALTER SESSION SET CURRENT_SCHEMA = testdb; +/ + +DECLARE + users_total NUMBER := 500000; + users_batch_size NUMBER := 5000; + users_current NUMBER := 0; + users_batch_end NUMBER; + -- about_text CLOB; +BEGIN + DBMS_OUTPUT.PUT_LINE('Inserting test data into testdb.users (' || users_total || ' rows)...'); + + WHILE users_current < users_total + LOOP + users_batch_end := users_current + users_batch_size; + IF users_batch_end > users_total THEN + users_batch_end := users_total; + END IF; + + -- Insert batch using a CTE-style approach + -- INSERT INTO testdb.users (name, surname, about, email, date_of_birth, join_date, created_at, is_active, login_count, balance) + INSERT INTO testdb.users (name, surname, email, date_of_birth, join_date, created_at, is_active, login_count, balance) + SELECT + 'user-' || n, -- name + 'surname-' || n, -- surname + -- RPAD('This is about user ' || n || '. ', 500000, 'X'), -- about ~500 KB + 'user' || n || '@example.com', -- email + SYSDATE - MOD(n, 10000), -- date_of_birth, spread over ~27 years + SYSTIMESTAMP, -- join_date + SYSTIMESTAMP, -- created_at + CASE WHEN MOD(n, 2) = 0 THEN 1 ELSE 0 END, -- is_active alternating 1/0 + MOD(n, 100), -- login_count between 0-99 + CAST(MOD(n, 1000) + MOD(n, 100) / 100.0 AS NUMBER(10,2)) -- balance + FROM ( + SELECT ROWNUM + users_current AS n + FROM dual + CONNECT BY LEVEL <= (users_batch_end - users_current) + ); + + COMMIT; + + users_current := users_batch_end; + + -- Log progress after every batch + DBMS_OUTPUT.PUT_LINE('Progress: ' || users_current || '/' || users_total || ' rows inserted into testdb.users'); + END LOOP; + + DBMS_OUTPUT.PUT_LINE('Completed: ' || users_current || ' rows inserted into testdb.users'); +END; +/ + +-- Verification +DECLARE + users_count NUMBER; +BEGIN + SELECT COUNT(*) INTO users_count FROM testdb.users; + DBMS_OUTPUT.PUT_LINE('Verification - testdb.users: ' || users_count || ' rows'); +END; +/