@@ -124,6 +124,17 @@ function help() {
124
124
125
125
PostgreSQL config to be used (may be partial).
126
126
127
+ \033[1m--pg-config-auto\033[22m (enum: oltp|olap)
128
+
129
+ Perform \" auto-tuning\" for PostgreSQL config. Allowed values:
130
+
131
+ * \" oltp\" to auto-tune Postgres for OLTP workload,
132
+ * \" olap\" to auto-tune Postgres for OLAP (analytical) workload.
133
+
134
+ This option can be combined with \" --pg-config\" – in this case, it will be
135
+ applied *after* it (so \" auto-tuning\" values will be added to the end of
136
+ the postgresql.conf file).
137
+
127
138
\033[1m--db-prepared-snapshot\033[22m (string)
128
139
129
140
Reserved / Not yet implemented.
@@ -304,6 +315,7 @@ function dbg_cli_parameters() {
304
315
echo " AWS_SSH_KEY_PATH: $AWS_SSH_KEY_PATH "
305
316
echo " PG_VERSION: ${PG_VERSION} "
306
317
echo " PG_CONFIG: ${PG_CONFIG} "
318
+ echo " PG_CONFIG_AUTO: ${PG_CONFIG_AUTO} "
307
319
echo " DB_PREPARED_SNAPSHOT: ${DB_PREPARED_SNAPSHOT} "
308
320
echo " DB_DUMP: $DB_DUMP "
309
321
echo " DB_NAME: $DB_NAME "
@@ -543,8 +555,11 @@ function check_cli_parameters() {
543
555
fi
544
556
545
557
if [[ -z ${PG_CONFIG+x} ]]; then
546
- err " NOTICE: No PostgreSQL config is provided. Will use default."
547
- # TODO(NikolayS) use "auto-tuning" – shared_buffers=1/4 RAM, etc
558
+ if [[ -z ${PG_CONFIG_AUTO+x} } ]]; then
559
+ err " NOTICE: No PostgreSQL config is provided. Will use default."
560
+ else
561
+ msg " Postgres config will be auto-tuned."
562
+ fi
548
563
else
549
564
check_path PG_CONFIG
550
565
if [[ " $? " -ne " 0" ]]; then # TODO(NikolayS) support file:// and s3://
@@ -941,6 +956,43 @@ function cleanup_and_exit {
941
956
fi
942
957
}
943
958
959
+ # ######################################
960
+ # Determine how many CPU, RAM we have, and what kind of disks.
961
+ # Globals:
962
+ # CPU_CNT, RAM_MB, DISK_ROTATIONAL
963
+ # Arguments:
964
+ # None
965
+ # Returns:
966
+ # None
967
+ # ######################################
968
+ function get_system_characteristics() {
969
+ # TODO(NikolayS) hyperthreading?
970
+ CPU_CNT=$( docker_exec bash -c " cat /proc/cpuinfo | grep processor | wc -l" )
971
+
972
+ local ram_bytes=$( \
973
+ docker_exec bash -c " cat /proc/meminfo | grep MemTotal | awk '{print \$ 2}'" \
974
+ )
975
+ RAM_MB=$( \
976
+ docker_exec bash -c \
977
+ " echo \" print round(\$ (cat /proc/meminfo | grep MemTotal | awk '{print \$ 2}').0 / 1000, 0)\" | python" \
978
+ )
979
+ # TODO(NikolayS) use bc instead of python
980
+
981
+ if [[ " $RUN_ON " == " aws" ]]; then
982
+ if [[ " ${AWS_EC2_TYPE: 0: 2} " == " i3" ]]; then
983
+ DISK_ROTATIONAL=false
984
+ else
985
+ DISK_ROTATIONAL=true # EBS might be SSD, but here we consider them as
986
+ # high-latency disks (TODO(NikolayS) improve
987
+ fi
988
+ else
989
+ # TODO(NikolayS) check if we work with SSD or not
990
+ DISK_ROTATIONAL=false
991
+ fi
992
+
993
+ msg " CPU_CNT: $CPU_CNT , RAM_MB: $RAM_MB , DISK_ROTATIONAL: $DISK_ROTATIONAL "
994
+ }
995
+
944
996
# ######################################
945
997
# # # # # MAIN # # # # #
946
998
# ######################################
@@ -966,6 +1018,8 @@ while [ $# -gt 0 ]; do
966
1018
PG_VERSION=" $2 " ; shift 2 ;;
967
1019
--pg-config )
968
1020
PG_CONFIG=" $2 " ; shift 2;;
1021
+ --pg-config-auto )
1022
+ PG_CONFIG_AUTO=" $2 " ; shift 2;;
969
1023
--db-prepared-snapshot )
970
1024
# Still unsupported
971
1025
DB_PREPARED_SNAPSHOT=" $2 " ; shift 2 ;;
1134
1188
MACHINE_HOME=" /machine_home/nancy_${CONTAINER_HASH} "
1135
1189
1136
1190
alias docker_exec=' docker $DOCKER_CONFIG exec -i ${CONTAINER_HASH} '
1137
- CPU_CNT= $( docker_exec bash -c " cat /proc/cpuinfo | grep processor | wc -l " ) # for execute in docker
1191
+ get_system_characteristics
1138
1192
1139
1193
docker_exec bash -c " mkdir $MACHINE_HOME && chmod a+w $MACHINE_HOME "
1140
1194
if [[ " $RUN_ON " == " aws" ]]; then
@@ -1336,26 +1390,75 @@ function apply_ddl_undo_code() {
1336
1390
# Returns:
1337
1391
# None
1338
1392
# ######################################
1339
- function apply_initial_postgres_configuration () {
1393
+ function pg_config_init () {
1340
1394
# Apply initial postgres configuration
1395
+ local restart_needed=false
1341
1396
OP_START_TIME=$( date +%s)
1342
- if ([ ! -z ${PG_CONFIG+x} ] && [ " $PG_CONFIG " != " " ]); then
1343
- msg " Apply initial postgres configuration "
1397
+ if ([[ ! -z ${PG_CONFIG+x} ]] && [[ " $PG_CONFIG " != " " ] ]); then
1398
+ msg " Initialize Postgres config (postgresql.conf). "
1344
1399
PG_CONFIG_FILENAME=$( basename $PG_CONFIG )
1345
1400
docker_exec bash -c " cat $MACHINE_HOME /$PG_CONFIG_FILENAME >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1346
- if [ -z ${DELTA_CONFIG+x} ]
1347
- then
1348
- docker_exec bash -c " sudo /etc/init.d/postgresql restart"
1349
- sleep 10
1401
+ restart_needed=true
1402
+ fi
1403
+ if [[ ! -z ${PG_CONFIG_AUTO+x} ]]; then
1404
+ msg " Auto-tune PostgreSQL (mode: '$PG_CONFIG_AUTO ')..."
1405
+ # TODO(NikolayS): better auto-tuning, more params
1406
+ # see:
1407
+ # - https://pgtune.leopard.in.ua
1408
+ # - https://postgresqlco.nf/ and https://github.com/jberkus/annotated.conf
1409
+ # - http://pgconfigurator.cybertec.at/
1410
+ # TODO(NikolayS): use bc instead of python (add bc to the docker image first)
1411
+ local shared_buffers=" $( echo " print round($RAM_MB / 4)" | python | awk -F ' .' ' {print $1}' ) MB"
1412
+ local effective_cache_size=" $( echo " print round(3 * $RAM_MB / 4)" | python | awk -F ' .' ' {print $1}' ) MB"
1413
+ if [[ " $PG_CONFIG_AUTO " = " oltp" ]]; then
1414
+ local work_mem=" $( echo " print round($RAM_MB / 5)" | python | awk -F ' .' ' {print $1}' ) kB"
1415
+ elif [[ " $PG_CONFIG_AUTO " = " olap" ]]; then
1416
+ local work_mem=" $( echo " print round($RAM_MB / 5)" | python | awk -F ' .' ' {print $1}' ) kB"
1417
+ else
1418
+ err " ASSERT: must not reach this point"
1419
+ exit 1
1350
1420
fi
1351
- END_TIME=$( date +%s)
1352
- DURATION=$( echo $(( END_TIME- OP_START_TIME)) | awk ' {printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}' )
1353
- msg " Time taken to apply Postgres initial configuration: $DURATION ."
1421
+ if [[ $work_mem = " 0kB" ]]; then # sanity check, set to tiny value just to start
1422
+ work_mem=" 1kB"
1423
+ fi
1424
+ if [[ $DISK_ROTATIONAL = false ]]; then
1425
+ local random_page_cost=" 1.1"
1426
+ local effective_io_concurrency=" 200"
1427
+ else
1428
+ local random_page_cost=" 4.0"
1429
+ local effective_io_concurrency=" 2"
1430
+ fi
1431
+ if [[ $CPU_CNT > 1 ]]; then # Only for postgres 9.6+!
1432
+ local max_worker_processes=" $CPU_CNT "
1433
+ local max_parallel_workers_per_gather=" $( echo " print round($CPU_CNT / 2)" | python | awk -F ' .' ' {print $1}' ) "
1434
+ local max_parallel_workers=" $CPU_CNT "
1435
+ fi
1436
+
1437
+ docker_exec bash -c " echo '# AUTO-TUNED KNOBS:' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1438
+ docker_exec bash -c " echo 'shared_buffers = $shared_buffers ' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1439
+ docker_exec bash -c " echo 'effective_cache_size = $effective_cache_size ' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1440
+ docker_exec bash -c " echo 'work_mem = $work_mem ' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1441
+ docker_exec bash -c " echo 'random_page_cost = $random_page_cost ' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1442
+ docker_exec bash -c " echo 'effective_io_concurrency = $effective_io_concurrency ' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1443
+ docker_exec bash -c " echo 'max_worker_processes = $max_worker_processes ' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1444
+ docker_exec bash -c " echo 'max_parallel_workers_per_gather = $max_parallel_workers_per_gather ' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1445
+ docker_exec bash -c " echo 'max_parallel_workers = $max_parallel_workers ' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1446
+ restart_needed=true
1447
+ fi
1448
+ if [[ ! -z ${DELTA_CONFIG+x} ]]; then # if DELTA_CONFIG is not empty, restart will be done later
1449
+ local restart_needed=false
1450
+ fi
1451
+ if [[ $restart_needed == true ]]; then
1452
+ docker_exec bash -c " sudo /etc/init.d/postgresql restart"
1453
+ sleep 10
1354
1454
fi
1455
+ END_TIME=$( date +%s)
1456
+ DURATION=$( echo $(( END_TIME- OP_START_TIME)) | awk ' {printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}' )
1457
+ msg " Time taken to apply Postgres initial configuration: $DURATION ."
1355
1458
}
1356
1459
1357
1460
# ######################################
1358
- # Apply test postgres configuration
1461
+ # Apply Postgres "delta" configuration
1359
1462
# Globals:
1360
1463
# DELTA_CONFIG, MACHINE_HOME, docker_exec alias
1361
1464
# Arguments:
@@ -1364,11 +1467,11 @@ function apply_initial_postgres_configuration() {
1364
1467
# None
1365
1468
# ######################################
1366
1469
function apply_postgres_configuration() {
1367
- # Apply postgres configuration
1368
1470
OP_START_TIME=$( date +%s)
1369
1471
if ([ ! -z ${DELTA_CONFIG+x} ] && [ " $DELTA_CONFIG " != " " ]); then
1370
- msg " Apply postgres configuration"
1472
+ msg " Apply configuration delta... "
1371
1473
DELTA_CONFIG_FILENAME=$( basename $DELTA_CONFIG )
1474
+ docker_exec bash -c " echo '# DELTA:' >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1372
1475
docker_exec bash -c " cat $MACHINE_HOME /$DELTA_CONFIG_FILENAME >> /etc/postgresql/$PG_VERSION /main/postgresql.conf"
1373
1476
docker_exec bash -c " sudo /etc/init.d/postgresql restart"
1374
1477
sleep 10
@@ -1466,21 +1569,21 @@ function collect_results() {
1466
1569
done
1467
1570
1468
1571
for table2export in \
1469
- " pg_stat_statements" \
1572
+ " pg_stat_statements order by total_time desc " \
1470
1573
" pg_stat_archiver" \
1471
1574
" pg_stat_bgwriter" \
1472
- " pg_stat_database" \
1473
- " pg_stat_database_conflicts" \
1474
- " pg_stat_all_tables" \
1475
- " pg_stat_xact_all_tables" \
1476
- " pg_stat_all_indexes" \
1477
- " pg_statio_all_tables" \
1478
- " pg_statio_all_indexes" \
1479
- " pg_statio_all_sequences" \
1480
- " pg_stat_user_functions" \
1481
- " pg_stat_xact_user_functions" \
1575
+ " pg_stat_database order by datname " \
1576
+ " pg_stat_database_conflicts order by datname " \
1577
+ " pg_stat_all_tables order by schemaname, relname " \
1578
+ " pg_stat_xact_all_tables order by schemaname, relname " \
1579
+ " pg_stat_all_indexes order by schemaname, relname, indexrelname " \
1580
+ " pg_statio_all_tables order by schemaname, relname " \
1581
+ " pg_statio_all_indexes order by schemaname, relname, indexrelname " \
1582
+ " pg_statio_all_sequences order by schemaname, relname " \
1583
+ " pg_stat_user_functions order by schemaname, funcname " \
1584
+ " pg_stat_xact_user_functions order by schemaname, funcname " \
1482
1585
; do
1483
- docker_exec bash -c " psql -U postgres $DB_NAME -b -c \" copy (select * from $table2export ) to stdout with csv header delimiter ',';\" > /$MACHINE_HOME /$ARTIFACTS_FILENAME /$ table2export .csv"
1586
+ docker_exec bash -c " psql -U postgres $DB_NAME -b -c \" copy (select * from $table2export ) to stdout with csv header delimiter ',';\" > /$MACHINE_HOME /$ARTIFACTS_FILENAME /\$ (echo \" $ table2export\" | awk '{print \$ 1}') .csv"
1484
1587
done
1485
1588
1486
1589
docker_exec bash -c " gzip -c $logpath > $MACHINE_HOME /$ARTIFACTS_FILENAME /postgresql.workload.log.gz"
@@ -1524,8 +1627,8 @@ apply_sql_before_db_restore
1524
1627
restore_dump
1525
1628
apply_sql_after_db_restore
1526
1629
docker_exec bash -c " psql -U postgres $DB_NAME -b -c 'create extension if not exists pg_stat_statements;' $VERBOSE_OUTPUT_REDIRECT "
1630
+ pg_config_init
1527
1631
apply_ddl_do_code
1528
- apply_initial_postgres_configuration
1529
1632
apply_postgres_configuration
1530
1633
prepare_start_workload
1531
1634
execute_workload
@@ -1554,4 +1657,4 @@ echo -e " Queries: "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILEN
1554
1657
echo -e " Query groups: " $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger.json | jq ' .normalyzed_info | length' )
1555
1658
echo -e " Errors: " $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger.json | jq ' .overall_stat.errors_number' )
1556
1659
echo -e " Errors groups: " $( docker_exec cat $MACHINE_HOME /$ARTIFACTS_FILENAME /pgbadger.json | jq ' .error_info | length' )
1557
- echo -e " ------------------------------------------------------------------------------"
1660
+ echo -e " ------------------------------------------------------------------------------"
0 commit comments