Add repmgr extension

.travis.yml

@@ -32,3 +32,14 @@ script:
 
 notifications:
   webhooks: https://galaxy.ansible.com/api/v1/notifications/
+
+jobs:
+  include:
+    - language: python
+      python: 2.7
+      env: IMAGE_NAME="centos:7-builded"
+      install:
+        - pip install ansible=="2.4.4.0" docker-py netaddr
+        - ln -s ${PWD} tests/docker/ANXS.postgresql
+      script:
+        - ansible-playbook -v -i tests/docker/hosts -e image_name=${IMAGE_NAME} tests/docker/replication.yml

README.md

@@ -52,6 +52,10 @@ Including an example of how to use your role:
 - :no_entry: - Has reached End of Life (EOL)
 
 
+#### Replication with repmgr
+
+There is initial support for setting up and running with replication managed by [repmgr](https://repmgr.org/). In it's current state it has only been tested with repmgr-4.2 and repmgr-5.0 on Centos 7 and requires Systemd.
+
 
 #### Variables
 
@@ -112,6 +116,36 @@ postgresql_user_privileges:
     db: foobar                  # database
     priv: "ALL"                 # privilege string format: example: INSERT,UPDATE/table:SELECT/anothertable:ALL
     role_attr_flags: "CREATEDB" # role attribute flags
+
+# Manage replication with repmgr (optional)
+repmgr_target_group: "postgresql-db"
+postgresql_ext_install_repmgr: yes
+repmgr_user: repmgr
+repmgr_database: repmgr
+postgresql_wal_level: "replica"
+postgresql_max_wal_senders: 10
+postgresql_max_replication_slots: 10
+postgresql_wal_keep_segments: 100
+postgresql_hot_standby: on
+postgresql_archive_mode: on
+postgresql_archive_command: "test ! -f /tmp/%f && cp %p /tmp/%f"
+postgresql_shared_preload_libraries:
+  - repmgr
+
+postgresql_users:
+  - name: "{{repmgr_user}}"
+    pass: "password"
+
+postgresql_databases:
+  - name: {{repmgr_database}}
+    owner: "{{repmgr_user}}"
+    encoding: "UTF-8"
+
+postgresql_user_privileges:
+  - name: "{{repmgr_user}}"
+    db: {{repmgr_database}}
+    priv: "ALL"
+    role_attr_flags: "SUPERUSER,REPLICATION"
 ```
 
 There's a lot more knobs and bolts to set, which you can find in the [defaults/main.yml](./defaults/main.yml)

defaults/main.yml

@@ -6,6 +6,7 @@ postgresql_version_terse: "{{ postgresql_version | replace('.', '') }}"  # Short
 postgresql_encoding: "UTF-8"
 postgresql_data_checksums: false
 postgresql_pwfile: ""
+repmgr_primary: True # Default to True to work without repmgr
 
 postgresql_locale_parts:
   - "en_US" # Locale
@@ -39,6 +40,7 @@ postgresql_database_owner: "{{ postgresql_admin_user }}"
 postgresql_ext_install_contrib: no
 postgresql_ext_install_dev_headers: no
 postgresql_ext_install_postgis: no
+postgresql_ext_install_repmgr: no
 
 postgresql_ext_postgis_version: "2.5" # be careful: check whether the postgresql/postgis versions work together
 

defaults/repmgr.yml

@@ -0,0 +1,116 @@
+# Role default overrides to work with replication
+postgresql_wal_level: "replica"
+postgresql_max_wal_senders: 10
+postgresql_max_replication_slots: 10
+postgresql_wal_keep_segments: 256 # Needs to be big enough for pg_basebackup to complete before this starts rotating files, 16MB each
+postgresql_hot_standby: on
+postgresql_archive_mode: off
+postgresql_archive_command: "/bin/true"
+postgresql_shared_preload_libraries:
+  - repmgr
+
+# Required configuration items
+repmgr_target_group: ""
+repmgr_user: "repmgr"
+repmgr_database: "repmgr"
+repmgr_conninfo: "host={{ansible_hostname}} user={{repmgr_user}} dbname={{repmgr_database}} connect_timeout=2"
+repmgr_data_directory: "{{postgresql_data_directory}}"
+repmgr_version: "4.2"
+
+# Optional configuration items
+#
+
+# Repositories
+repmgr_gpg_key_url: "https://dl.2ndquadrant.com/gpg-key.asc"
+repmgr_baseurl: "https://dl.2ndquadrant.com/default/release/rpm/packages/centos/$releasever/$basearch/{{postgresql_version}}/"
+repmgr_apt_repository: "deb [arch=amd64] https://dl.2ndquadrant.com/default/release/apt {{ ansible_distribution_release }}-2ndquadrant main"
+
+# Replication
+repmgr_primary: False
+repmgr_config_directory: "{{postgresql_conf_directory}}"
+repmgr_replication_user: "{{repmgr_user}}"
+repmgr_replication_type: "physical"
+repmgr_location: "default"
+repmgr_use_replication_slots: "no"
+repmgr_witness_sync_interval: 15
+repmgr_use_primary_conninfo_password: "false"
+repmgr_passfile: "/var/lib/pgsql/.pgpass"
+repmgr_private_key_path: "/var/lib/pgsql/.ssh/id_rsa"
+repmgr_public_key_path: "/var/lib/pgsql/.ssh/id_rsa.pub"
+repmgr_ssh_key_bits: "2048"
+repmgr_ssh_key_type: "rsa"
+
+# Logging
+repmgr_log_level: "INFO"
+repmgr_log_facility: "STDERR"
+repmgr_log_file: ""
+repmgr_log_status_interval: 300
+
+# Notification
+repmgr_event_notification_command: ""
+repmgr_event_notifications: ""
+
+# Directories
+repmgr_pg_bindir: "{{postgresql_bin_directory}}"
+repmgr_repmgr_bindir: "{{postgresql_bin_directory}}"
+
+# Commands & options
+repmgr_pg_ctl_options: ""
+repmgr_pg_basebackup_options: ""
+repmgr_rsync_options: ""
+repmgr_tablespace_mapping: ""
+repmgr_restore_command: ""
+repmgr_archive_cleanup_command: ""
+repmgr_recovery_min_apply_delay: ""
+repmgr_ssh_options: "-o StrictHostKeyChecking=no"
+
+# Intervals & timeouts
+repmgr_promote_check_timeout: 60
+repmgr_promote_check_interval: 1
+repmgr_primary_follow_timeout: 60
+repmgr_standby_follow_timeout: 15
+repmgr_shutdown_check_timeout: 60
+repmgr_standby_reconnect_timeout: 60
+repmgr_node_rejoin_timeout: 60
+
+# Barman
+repmgr_barman_server: ""
+repmgr_barman_host: ""
+repmgr_barman_config: ""
+repmgr_repmgrd_pid_file: ""
+
+# Fail-over
+repmgr_failover: "manual"
+repmgr_standby_disconnect_on_failover: "false"
+repmgr_primary_visibility_consensus: "false"
+repmgr_priority: 100
+repmgr_reconnect_attempts: 6
+repmgr_reconnect_interval: 10
+repmgr_promote_command: ""
+repmgr_follow_command: ""
+repmgr_primary_notification_timeout: 60
+repmgr_repmgrd_standby_startup_timeout: 60
+
+# Monitoring
+repmgr_monitoring_history: "no"
+repmgr_monitor_interval_secs: 2
+repmgr_degraded_monitoring_timeout: -1
+repmgr_async_query_timeout: 60
+repmgr_keep_history_days: 14
+
+# Service commands
+repmgr_service_start_command :  "sudo systemctl start postgresql-{{postgresql_version}}"
+repmgr_service_stop_command :  "sudo systemctl stop postgresql-{{postgresql_version}}"
+repmgr_service_restart_command :  "sudo systemctl restart postgresql-{{postgresql_version}}"
+repmgr_service_reload_command :  "sudo systemctl reload postgresql-{{postgresql_version}}"
+repmgr_service_promote_command :  ""
+
+# Warning levels
+repmgr_archive_ready_warning: 16
+repmgr_archive_ready_critical: 128
+repmgr_replication_lag_warning: 300
+repmgr_replication_lag_critical: 600
+
+# Bi-Directional Replication
+repmgr_bdr_local_monitoring_only: "false"
+repmgr_bdr_recovery_timeout: 30

handlers/main.yml

@@ -1,7 +1,15 @@
+---
+# vim: set syntax=yaml expandtab shiftwidth=2 tabstop=2 softtabstop=2 autoindent smartindent:
 # file: postgresql/handlers/main.yml
 
   - name: restart postgresql
     service:
       name: "{{ postgresql_service_name }}"
       state: restarted
       enabled: yes
+
+  - name: restart repmgr
+    service:
+      name: "repmgr{{postgresql_version}}"
+      state: restarted
+      enabled: yes

tasks/databases.yml

@@ -19,7 +19,7 @@
   become: yes
   become_user: "{{postgresql_admin_user}}"
   with_items: "{{postgresql_databases}}"
-  when: postgresql_databases|length > 0
+  when: postgresql_databases|length > 0 and repmgr_primary
 
 - name: PostgreSQL | Add extensions to the databases
   become: yes
@@ -33,6 +33,7 @@
     - "{{ postgresql_database_extensions }}"
     - extensions
   register: result
+  when: repmgr_primary
 
 - name: PostgreSQL | Add hstore to the databases with the requirement
   become: yes

tasks/extensions.yml

@@ -6,3 +6,5 @@
   when: postgresql_ext_install_dev_headers
 - import_tasks: extensions/postgis.yml
   when: postgresql_ext_install_postgis
+- import_tasks: extensions/repmgr.yml
+  when: postgresql_ext_install_repmgr and ansible_service_mgr == 'systemd'

tasks/extensions/configure_repmgr.yml

@@ -0,0 +1,149 @@
+- name: Repmgr | Update configuration (repmgr.conf)
+  template:
+    src: "repmgr.conf-{{ repmgr_version[:3] }}.j2"
+    dest: "{{postgresql_conf_directory}}/repmgr.conf"
+    owner: "{{ postgresql_service_user }}"
+    group: "{{ postgresql_service_group }}"
+    mode: 0640
+  notify: restart repmgr
+
+- name: Repmgr | Ensure systemd drop-in directory exists
+  file:
+    path: "/etc/systemd/system/repmgr{{postgresql_version}}.service.d/"
+    state: directory
+    mode: 0755
+
+- name: Repmgr | Update drop-in
+  template:
+    src: "repmgr.custom.conf.j2"
+    dest: "/etc/systemd/system/repmgr{{postgresql_version}}.service.d/custom.conf"
+
+- name: Repmgr | Allow passwordless restarts with postgres user
+  template:
+    src: "sudoers.postgresql.j2"
+    dest: "/etc/sudoers.d/postgresql"
+    mode: 0640
+
+- name: Repmgr | Update .pgpass for postgres user
+  template:
+    src: "pgpass.j2"
+    dest: "{{repmgr_passfile}}"
+    owner: "{{ postgresql_service_user }}"
+    group: "{{ postgresql_service_group }}"
+    mode: 0400
+    trim_blocks: no
+
+- name: Repmgr | Generate ssh keypair for postgres user
+  command: "ssh-keygen -b {{repmgr_ssh_key_bits}} -t {{repmgr_ssh_key_type}} -f {{repmgr_private_key_path}} -N ''"
+  args:
+    creates: "{{repmgr_private_key_path}}"
+  become: yes
+  become_user: "{{ postgresql_service_user }}"
+  when: repmgr_primary
+
+- name: Repmgr | Slurp ssh public key
+  slurp:
+    src: "{{repmgr_public_key_path}}"
+  register: ssh_public_key
+  when: repmgr_primary
+
+- name: Repmgr | Slurp ssh private key
+  slurp:
+    src: "{{repmgr_private_key_path}}"
+  register: ssh_private_key
+  when: repmgr_primary
+
+- name: Repmgr | Update authorized_keys for postgres user
+  authorized_key:
+    user: postgres
+    state: present
+    key: "{% for host, vars in hostvars.items() if 'repmgr_primary' in vars and vars['repmgr_primary'] == True %}{{ vars['ssh_public_key']['content'] | b64decode }}{% endfor %}"
+
+- name: Repmgr | Update public ssh key for postgres user
+  copy:
+    content: "{% for host, vars in hostvars.items() if 'repmgr_primary' in vars and vars['repmgr_primary'] == True %}{{ vars['ssh_public_key']['content'] | b64decode }}{% endfor %}"
+    dest: "{{repmgr_public_key_path}}"
+    owner: "{{ postgresql_service_user }}"
+    group: "{{ postgresql_service_group }}"
+    mode: 0644
+
+- name: Repmgr | Update private ssh key for postgres user
+  copy:
+    content: "{% for host, vars in hostvars.items() if 'repmgr_primary' in vars and vars['repmgr_primary'] == True %}{{ vars['ssh_private_key']['content'] | b64decode }}{% endfor %}"
+    dest: "{{repmgr_private_key_path}}"
+    owner: "{{ postgresql_service_user }}"
+    group: "{{ postgresql_service_group }}"
+    mode: 0600
+
+- name: Repmgr | Check cluster status
+  command: "{{postgresql_bin_directory}}/repmgr -f {{postgresql_conf_directory}}/repmgr.conf cluster show"
+  become: yes
+  become_user: "{{ postgresql_service_user }}"
+  changed_when: repmgr_cluster_show.rc != 0
+  register: repmgr_cluster_show
+  ignore_errors: True
+
+- name: Repmgr | Register as primary
+  command: "{{postgresql_bin_directory}}/repmgr -f {{postgresql_conf_directory}}/repmgr.conf primary register"
+  become: yes
+  become_user: "{{ postgresql_service_user }}"
+  when: repmgr_primary and not ansible_hostname in repmgr_cluster_show.stdout and not "primary" in repmgr_cluster_show.stdout
+
+- name: Repmgr | Ensure postgresql slave is stopped before clone
+  service:
+    name: "postgresql-{{postgresql_version}}"
+    state: stopped
+  when: not repmgr_primary and not ansible_hostname in repmgr_cluster_show.stdout and not "standby" in repmgr_cluster_show.stdout
+
+- name: Repmgr | Clone standby
+  command: "{{postgresql_bin_directory}}/repmgr -F -h {% for host, vars in hostvars.items() if 'repmgr_primary' in vars and vars['repmgr_primary'] == True %}{{ host }}{% endfor %} -U repmgr -d repmgr -f {{postgresql_conf_directory}}/repmgr.conf standby clone"
+  become: yes
+  become_user: "{{ postgresql_service_user }}"
+  when: not repmgr_primary and not ansible_hostname in  repmgr_cluster_show.stdout and not "standby" in repmgr_cluster_show.stdout
+
+- name: Repmgr | Ensure postgresql slave is running after clone
+  service:
+    name: "postgresql-{{postgresql_version}}"
+    state: started
+  when: not repmgr_primary
+
+- name: Repmgr | Wait for Postgres
+  wait_for:
+    timeout: 2
+  delegate_to: localhost
+
+- name: Repmgr | Register standby
+  command: "{{postgresql_bin_directory}}/repmgr -F -h {% for host, vars in hostvars.items() if 'repmgr_primary' in vars and vars['repmgr_primary'] == True %}{{ host }}{% endfor %} -U repmgr -d repmgr -f {{postgresql_conf_directory}}/repmgr.conf standby register"
+  become: yes
+  become_user: "{{ postgresql_service_user }}"
+  when: not repmgr_primary and not ansible_hostname in repmgr_cluster_show.stdout and not "standby" in repmgr_cluster_show.stdout
+
+- name: Repmgr | Verify cluster functionality
+  command: "{{postgresql_bin_directory}}/repmgr -F -h {% for host, vars in hostvars.items() if 'repmgr_primary' in vars and vars['repmgr_primary'] == True %}{{ host }}{% endfor %} -U repmgr -d repmgr -f {{postgresql_conf_directory}}/repmgr.conf cluster crosscheck"
+  become: yes
+  become_user: "{{ postgresql_service_user }}"
+
+- name: Repmgr | Ensure repmgrd is running
+  service:
+    name: "repmgr{{postgresql_version}}"
+    state: started
+    enabled: yes
+  when: repmgr_monitoring_history == "true" or repmgr_failover == "automatic"
+
+- name: Repmgr | Ensure crontab is installed
+  package:
+    name: crontabs
+  when: repmgr_monitoring_history == "true" or repmgr_failover == "automatic"
+
+- name: Repmgr | Ensure crond is running
+  service:
+    name: "crond"
+    state: started
+    enabled: yes
+
+- name: Repmgr | Setup cluster monitoring history cleanup
+  copy:
+    content: "{{postgresql_bin_directory}}/repmgr -f {{postgresql_conf_directory}}/repmgr.conf cluster cleanup --keep-history={{repmgr_keep_history_days}}"
+    dest: "/etc/cron.daily/repmgr_cleanup"
+    mode: 0755
+  when: repmgr_monitoring_history == "true" or repmgr_failover == "automatic"