Skip to content
This repository was archived by the owner on Aug 16, 2021. It is now read-only.

Commit 24de0a3

Browse files
NikolaySNikolayS
committed
Basic auto-tuning for postgresql.conf
Add get_system_characteristics function This function automatically detects, how many CPU cores are available to the Docker container, huw much GBs of RAM, and whether disks are "rotational" (HDD), or not (SSD). Basic autotuning steps Add basic autotuning (only 'oltp' at the moment) for knobs: - shared_buffers - effective_cache_size - random_page_cost Fix order of actions when changing Postgres config There was a logical error in determining, if Postgres restart is needed. Add order to pg_stat_*** artifacts (CSV files) fix CSV filename More auto-tuning - olap mode - more params, including parallel execution
1 parent 9a8b387 commit 24de0a3

File tree

1 file changed

+133
-30
lines changed

1 file changed

+133
-30
lines changed

‎nancy_run.sh

Lines changed: 133 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,17 @@ function help() {
124124
125125
PostgreSQL config to be used (may be partial).
126126
127+
\033[1m--pg-config-auto\033[22m (enum: oltp|olap)
128+
129+
Perform \"auto-tuning\" for PostgreSQL config. Allowed values:
130+
131+
* \"oltp\" to auto-tune Postgres for OLTP workload,
132+
* \"olap\" to auto-tune Postgres for OLAP (analytical) workload.
133+
134+
This option can be combined with \"--pg-config\" – in this case, it will be
135+
applied *after* it (so \"auto-tuning\" values will be added to the end of
136+
the postgresql.conf file).
137+
127138
\033[1m--db-prepared-snapshot\033[22m (string)
128139
129140
Reserved / Not yet implemented.
@@ -304,6 +315,7 @@ function dbg_cli_parameters() {
304315
echo "AWS_SSH_KEY_PATH: $AWS_SSH_KEY_PATH"
305316
echo "PG_VERSION: ${PG_VERSION}"
306317
echo "PG_CONFIG: ${PG_CONFIG}"
318+
echo "PG_CONFIG_AUTO: ${PG_CONFIG_AUTO}"
307319
echo "DB_PREPARED_SNAPSHOT: ${DB_PREPARED_SNAPSHOT}"
308320
echo "DB_DUMP: $DB_DUMP"
309321
echo "DB_NAME: $DB_NAME"
@@ -543,8 +555,11 @@ function check_cli_parameters() {
543555
fi
544556

545557
if [[ -z ${PG_CONFIG+x} ]]; then
546-
err "NOTICE: No PostgreSQL config is provided. Will use default."
547-
# TODO(NikolayS) use "auto-tuning" – shared_buffers=1/4 RAM, etc
558+
if [[ -z ${PG_CONFIG_AUTO+x}} ]]; then
559+
err "NOTICE: No PostgreSQL config is provided. Will use default."
560+
else
561+
msg "Postgres config will be auto-tuned."
562+
fi
548563
else
549564
check_path PG_CONFIG
550565
if [[ "$?" -ne "0" ]]; then # TODO(NikolayS) support file:// and s3://
@@ -941,6 +956,43 @@ function cleanup_and_exit {
941956
fi
942957
}
943958

959+
#######################################
960+
# Determine how many CPU, RAM we have, and what kind of disks.
961+
# Globals:
962+
# CPU_CNT, RAM_MB, DISK_ROTATIONAL
963+
# Arguments:
964+
# None
965+
# Returns:
966+
# None
967+
#######################################
968+
function get_system_characteristics() {
969+
#TODO(NikolayS) hyperthreading?
970+
CPU_CNT=$(docker_exec bash -c "cat /proc/cpuinfo | grep processor | wc -l")
971+
972+
local ram_bytes=$( \
973+
docker_exec bash -c "cat /proc/meminfo | grep MemTotal | awk '{print \$2}'" \
974+
)
975+
RAM_MB=$( \
976+
docker_exec bash -c \
977+
"echo \"print round(\$(cat /proc/meminfo | grep MemTotal | awk '{print \$2}').0 / 1000, 0)\" | python" \
978+
)
979+
#TODO(NikolayS) use bc instead of python
980+
981+
if [[ "$RUN_ON" == "aws" ]]; then
982+
if [[ "${AWS_EC2_TYPE:0:2}" == "i3" ]]; then
983+
DISK_ROTATIONAL=false
984+
else
985+
DISK_ROTATIONAL=true # EBS might be SSD, but here we consider them as
986+
# high-latency disks (TODO(NikolayS) improve
987+
fi
988+
else
989+
#TODO(NikolayS) check if we work with SSD or not
990+
DISK_ROTATIONAL=false
991+
fi
992+
993+
msg "CPU_CNT: $CPU_CNT, RAM_MB: $RAM_MB, DISK_ROTATIONAL: $DISK_ROTATIONAL"
994+
}
995+
944996
#######################################
945997
# # # # # MAIN # # # # #
946998
#######################################
@@ -966,6 +1018,8 @@ while [ $# -gt 0 ]; do
9661018
PG_VERSION="$2"; shift 2 ;;
9671019
--pg-config )
9681020
PG_CONFIG="$2"; shift 2;;
1021+
--pg-config-auto )
1022+
PG_CONFIG_AUTO="$2"; shift 2;;
9691023
--db-prepared-snapshot )
9701024
#Still unsupported
9711025
DB_PREPARED_SNAPSHOT="$2"; shift 2 ;;
@@ -1134,7 +1188,7 @@ fi
11341188
MACHINE_HOME="/machine_home/nancy_${CONTAINER_HASH}"
11351189

11361190
alias docker_exec='docker $DOCKER_CONFIG exec -i ${CONTAINER_HASH} '
1137-
CPU_CNT=$(docker_exec bash -c "cat /proc/cpuinfo | grep processor | wc -l") # for execute in docker
1191+
get_system_characteristics
11381192

11391193
docker_exec bash -c "mkdir $MACHINE_HOME && chmod a+w $MACHINE_HOME"
11401194
if [[ "$RUN_ON" == "aws" ]]; then
@@ -1336,26 +1390,75 @@ function apply_ddl_undo_code() {
13361390
# Returns:
13371391
# None
13381392
#######################################
1339-
function apply_initial_postgres_configuration() {
1393+
function pg_config_init() {
13401394
# Apply initial postgres configuration
1395+
local restart_needed=false
13411396
OP_START_TIME=$(date +%s)
1342-
if ([ ! -z ${PG_CONFIG+x} ] && [ "$PG_CONFIG" != "" ]); then
1343-
msg "Apply initial postgres configuration"
1397+
if ([[ ! -z ${PG_CONFIG+x} ]] && [[ "$PG_CONFIG" != "" ]]); then
1398+
msg "Initialize Postgres config (postgresql.conf)."
13441399
PG_CONFIG_FILENAME=$(basename $PG_CONFIG)
13451400
docker_exec bash -c "cat $MACHINE_HOME/$PG_CONFIG_FILENAME >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1346-
if [ -z ${DELTA_CONFIG+x} ]
1347-
then
1348-
docker_exec bash -c "sudo /etc/init.d/postgresql restart"
1349-
sleep 10
1401+
restart_needed=true
1402+
fi
1403+
if [[ ! -z ${PG_CONFIG_AUTO+x} ]]; then
1404+
msg "Auto-tune PostgreSQL (mode: '$PG_CONFIG_AUTO')..."
1405+
# TODO(NikolayS): better auto-tuning, more params
1406+
# see:
1407+
# - https://pgtune.leopard.in.ua
1408+
# - https://postgresqlco.nf/ and https://github.com/jberkus/annotated.conf
1409+
# - http://pgconfigurator.cybertec.at/
1410+
# TODO(NikolayS): use bc instead of python (add bc to the docker image first)
1411+
local shared_buffers="$(echo "print round($RAM_MB / 4)" | python | awk -F '.' '{print $1}')MB"
1412+
local effective_cache_size="$(echo "print round(3 * $RAM_MB / 4)" | python | awk -F '.' '{print $1}')MB"
1413+
if [[ "$PG_CONFIG_AUTO" = "oltp" ]]; then
1414+
local work_mem="$(echo "print round($RAM_MB / 5)" | python | awk -F '.' '{print $1}')kB"
1415+
elif [[ "$PG_CONFIG_AUTO" = "olap" ]]; then
1416+
local work_mem="$(echo "print round($RAM_MB / 5)" | python | awk -F '.' '{print $1}')kB"
1417+
else
1418+
err "ASSERT: must not reach this point"
1419+
exit 1
13501420
fi
1351-
END_TIME=$(date +%s)
1352-
DURATION=$(echo $((END_TIME-OP_START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
1353-
msg "Time taken to apply Postgres initial configuration: $DURATION."
1421+
if [[ $work_mem = "0kB" ]]; then # sanity check, set to tiny value just to start
1422+
work_mem="1kB"
1423+
fi
1424+
if [[ $DISK_ROTATIONAL = false ]]; then
1425+
local random_page_cost="1.1"
1426+
local effective_io_concurrency="200"
1427+
else
1428+
local random_page_cost="4.0"
1429+
local effective_io_concurrency="2"
1430+
fi
1431+
if [[ $CPU_CNT > 1 ]]; then # Only for postgres 9.6+!
1432+
local max_worker_processes="$CPU_CNT"
1433+
local max_parallel_workers_per_gather="$(echo "print round($CPU_CNT / 2)" | python | awk -F '.' '{print $1}')"
1434+
local max_parallel_workers="$CPU_CNT"
1435+
fi
1436+
1437+
docker_exec bash -c "echo '# AUTO-TUNED KNOBS:' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1438+
docker_exec bash -c "echo 'shared_buffers = $shared_buffers' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1439+
docker_exec bash -c "echo 'effective_cache_size = $effective_cache_size' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1440+
docker_exec bash -c "echo 'work_mem = $work_mem' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1441+
docker_exec bash -c "echo 'random_page_cost = $random_page_cost' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1442+
docker_exec bash -c "echo 'effective_io_concurrency = $effective_io_concurrency' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1443+
docker_exec bash -c "echo 'max_worker_processes = $max_worker_processes' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1444+
docker_exec bash -c "echo 'max_parallel_workers_per_gather = $max_parallel_workers_per_gather' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1445+
docker_exec bash -c "echo 'max_parallel_workers = $max_parallel_workers' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
1446+
restart_needed=true
1447+
fi
1448+
if [[ ! -z ${DELTA_CONFIG+x} ]]; then # if DELTA_CONFIG is not empty, restart will be done later
1449+
local restart_needed=false
1450+
fi
1451+
if [[ $restart_needed == true ]]; then
1452+
docker_exec bash -c "sudo /etc/init.d/postgresql restart"
1453+
sleep 10
13541454
fi
1455+
END_TIME=$(date +%s)
1456+
DURATION=$(echo $((END_TIME-OP_START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
1457+
msg "Time taken to apply Postgres initial configuration: $DURATION."
13551458
}
13561459

13571460
#######################################
1358-
# Apply test postgres configuration
1461+
# Apply Postgres "delta" configuration
13591462
# Globals:
13601463
# DELTA_CONFIG, MACHINE_HOME, docker_exec alias
13611464
# Arguments:
@@ -1364,11 +1467,11 @@ function apply_initial_postgres_configuration() {
13641467
# None
13651468
#######################################
13661469
function apply_postgres_configuration() {
1367-
# Apply postgres configuration
13681470
OP_START_TIME=$(date +%s)
13691471
if ([ ! -z ${DELTA_CONFIG+x} ] && [ "$DELTA_CONFIG" != "" ]); then
1370-
msg "Apply postgres configuration"
1472+
msg "Apply configuration delta..."
13711473
DELTA_CONFIG_FILENAME=$(basename $DELTA_CONFIG)
1474+
docker_exec bash -c "echo '# DELTA:' >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
13721475
docker_exec bash -c "cat $MACHINE_HOME/$DELTA_CONFIG_FILENAME >> /etc/postgresql/$PG_VERSION/main/postgresql.conf"
13731476
docker_exec bash -c "sudo /etc/init.d/postgresql restart"
13741477
sleep 10
@@ -1466,21 +1569,21 @@ function collect_results() {
14661569
done
14671570

14681571
for table2export in \
1469-
"pg_stat_statements" \
1572+
"pg_stat_statements order by total_time desc" \
14701573
"pg_stat_archiver" \
14711574
"pg_stat_bgwriter" \
1472-
"pg_stat_database" \
1473-
"pg_stat_database_conflicts" \
1474-
"pg_stat_all_tables" \
1475-
"pg_stat_xact_all_tables" \
1476-
"pg_stat_all_indexes" \
1477-
"pg_statio_all_tables" \
1478-
"pg_statio_all_indexes" \
1479-
"pg_statio_all_sequences" \
1480-
"pg_stat_user_functions" \
1481-
"pg_stat_xact_user_functions" \
1575+
"pg_stat_database order by datname" \
1576+
"pg_stat_database_conflicts order by datname" \
1577+
"pg_stat_all_tables order by schemaname, relname" \
1578+
"pg_stat_xact_all_tables order by schemaname, relname" \
1579+
"pg_stat_all_indexes order by schemaname, relname, indexrelname" \
1580+
"pg_statio_all_tables order by schemaname, relname" \
1581+
"pg_statio_all_indexes order by schemaname, relname, indexrelname" \
1582+
"pg_statio_all_sequences order by schemaname, relname" \
1583+
"pg_stat_user_functions order by schemaname, funcname" \
1584+
"pg_stat_xact_user_functions order by schemaname, funcname" \
14821585
; do
1483-
docker_exec bash -c "psql -U postgres $DB_NAME -b -c \"copy (select * from $table2export) to stdout with csv header delimiter ',';\" > /$MACHINE_HOME/$ARTIFACTS_FILENAME/$table2export.csv"
1586+
docker_exec bash -c "psql -U postgres $DB_NAME -b -c \"copy (select * from $table2export) to stdout with csv header delimiter ',';\" > /$MACHINE_HOME/$ARTIFACTS_FILENAME/\$(echo \"$table2export\" | awk '{print \$1}').csv"
14841587
done
14851588

14861589
docker_exec bash -c "gzip -c $logpath > $MACHINE_HOME/$ARTIFACTS_FILENAME/postgresql.workload.log.gz"
@@ -1524,8 +1627,8 @@ apply_sql_before_db_restore
15241627
restore_dump
15251628
apply_sql_after_db_restore
15261629
docker_exec bash -c "psql -U postgres $DB_NAME -b -c 'create extension if not exists pg_stat_statements;' $VERBOSE_OUTPUT_REDIRECT"
1630+
pg_config_init
15271631
apply_ddl_do_code
1528-
apply_initial_postgres_configuration
15291632
apply_postgres_configuration
15301633
prepare_start_workload
15311634
execute_workload
@@ -1554,4 +1657,4 @@ echo -e " Queries: "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILEN
15541657
echo -e " Query groups: "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.normalyzed_info | length')
15551658
echo -e " Errors: "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.errors_number')
15561659
echo -e " Errors groups: "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.error_info | length')
1557-
echo -e "------------------------------------------------------------------------------"
1660+
echo -e "------------------------------------------------------------------------------"

0 commit comments

Comments
 (0)